diff --git a/.gitignore b/.gitignore index 3eda69f60de91638ef0190283d092e9b6e0229a2..1bfbf00cd52ce01a68de6aee8bfdd07637e46b65 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,7 @@ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ source/libs/parser/inc/sql.* +tests/script/tmqResult.txt # Emacs # -*- mode: gitignore; -*- diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9cf68b87f9764b4670cf7091c2f7d83bbb347063..575a0e6274691d3b24429a891979507a36eed543 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -83,11 +83,6 @@ if(${BUILD_WITH_NURAFT}) cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_NURAFT}) -# iconv -if(${BUILD_WITH_ICONV}) - cat("${CMAKE_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif(${BUILD_WITH_ICONV}) - # download dependencies configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -213,10 +208,9 @@ endif(${BUILD_WITH_TRAFT}) # LIBUV if(${BUILD_WITH_UV}) - if (${TD_WINDOWS}) - file(READ "libuv/include/uv.h" CONTENTS) - string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}") - file(WRITE "libuv/include/uv.h" "${CONTENTS_NEW}") + if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") + MESSAGE("Windows need set no-sign-compare") + add_compile_options(-Wno-sign-compare) endif () add_subdirectory(libuv) endif(${BUILD_WITH_UV}) @@ -249,15 +243,7 @@ if(${BUILD_WITH_SQLITE}) endif(${BUILD_WITH_SQLITE}) # pthread -if(${BUILD_PTHREAD}) - add_definitions(-DPTW32_STATIC_LIB) - add_subdirectory(pthread) -endif(${BUILD_PTHREAD}) - -# iconv -if(${BUILD_WITH_ICONV}) - add_subdirectory(iconv) -endif(${BUILD_WITH_ICONV}) + # ================================================================================================ # Build test diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 51e1e996dc2d6f634e185a843a71bb3c24d91634..acb99caffc5b6ab48fb9fdffc237bf1749d5a869 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -1,12 +1,30 @@ -aux_source_directory(src TMQ_DEMO_SRC) +add_executable(tmq "") +add_executable(tstream "") -add_executable(tmq ${TMQ_DEMO_SRC}) -target_link_libraries( - tmq taos +target_sources(tmq + PRIVATE + "src/tmq.c" ) -target_include_directories( - tmq + +target_sources(tstream + PRIVATE + "src/tstream.c" +) +target_link_libraries(tmq + taos +) + +target_link_libraries(tstream + taos +) + +target_include_directories(tmq + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_include_directories(tstream PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) +SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream) diff --git a/example/src/tstream.c b/example/src/tstream.c new file mode 100644 index 0000000000000000000000000000000000000000..8ffa932bd29d6f89441c7f045611f42e058dd872 --- /dev/null +++ b/example/src/tstream.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "taos.h" + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; +} + +int32_t create_stream() { + printf("create stream\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/ + const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1"; + /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ + pRes = tmq_create_stream(pConn, "stream1", "out1", sql); + if (taos_errno(pRes) != 0) { + printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + taos_close(pConn); + return 0; +} + +int main(int argc, char* argv[]) { + int code; + if (argc > 1) { + printf("env init\n"); + code = init_env(); + } + create_stream(); +#if 0 + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + /*perf_loop(tmq, topic_list);*/ + /*basic_consume_loop(tmq, topic_list);*/ + sync_consume_loop(tmq, topic_list); +#endif +} diff --git a/include/client/taos.h b/include/client/taos.h index a102629efa425ca71143669f18091eef073fe414..82f0635612a75ac6b26e5c28f2ba21f3b2f6709f 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -85,11 +85,7 @@ typedef struct taosField { int32_t bytes; } TAOS_FIELD; -#ifdef _TD_GO_DLL_ -#define DLL_EXPORT __declspec(dllexport) -#else #define DLL_EXPORT -#endif typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); @@ -218,7 +214,6 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, v DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); -DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); @@ -262,7 +257,12 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message); -/* ---------------------- OTHER ---------------------------- */ +/* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ +DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); + +DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql); + +/* -------------------------------- OTHER -------------------------------- */ typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 39d08f2e868667296c3fd39961cb3c080b48ab2a..797ebf29c59c38bc4fd5af1143976037360843b1 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -30,12 +30,13 @@ typedef int64_t tb_uid_t; #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) typedef enum { - TSDB_SUPER_TABLE = 1, // super table - TSDB_CHILD_TABLE = 2, // table created from super table - TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created by stream processing - TSDB_TEMP_TABLE = 5, // temp table created by nest query - TSDB_TABLE_MAX = 6 + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_SYSTEM_TABLE = 6, + TSDB_TABLE_MAX = 7 } ETableType; typedef enum { @@ -62,9 +63,12 @@ typedef enum { } ETsdbStatisStatus; typedef enum { - TSDB_SMA_STAT_OK = 0, // ready to provide service - TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired -} ETsdbSmaStat; + TSDB_SMA_STAT_UNKNOWN = -1, // unknown + TSDB_SMA_STAT_OK = 0, // ready to provide service + TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired + TSDB_SMA_STAT_DROPPED = 2, // sma dropped +} ETsdbSmaStat; // bit operation + typedef enum { TSDB_SMA_TYPE_BLOCK = 0, // Block-wise SMA diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 385c123fec8f87c7c0785916139d2082e273f33c..eb9f45087295841fbf03e580955d430f510522f0 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -54,25 +54,21 @@ typedef struct SColumnDataAgg { } SColumnDataAgg; typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t rowSize; - int16_t numOfCols; - int16_t hasVarCol; - union {int64_t uid; int64_t blockId;}; + STimeWindow window; + int32_t rows; + int32_t rowSize; + int16_t numOfCols; + int16_t hasVarCol; + union { + int64_t uid; + int64_t blockId; + }; + int64_t groupId; // no need to serialize } SDataBlockInfo; -//typedef struct SConstantItem { -// SColumnInfo info; -// int32_t startRow; // run-length-encoding to save the space for multiple rows -// int32_t endRow; -// SVariant value; -//} SConstantItem; - -// info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg *pBlockAgg; - SArray *pDataBlock; // SArray + SColumnDataAgg* pBlockAgg; + SArray* pDataBlock; // SArray SDataBlockInfo info; } SSDataBlock; @@ -98,23 +94,40 @@ void* blockDataDestroy(SSDataBlock* pBlock); int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); -static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { - if (pBlock == NULL) { - return; +int32_t tEncodeDataBlocks(void** buf, const SArray* blocks); +void* tDecodeDataBlocks(const void* buf, SArray** blocks); + +static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { + // WARNING: do not use info.numOfCols, + // sometimes info.numOfCols != array size + int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + taosMemoryFreeClear(pColInfoData->varmeta.offset); + } else { + taosMemoryFreeClear(pColInfoData->nullbitmap); + } + + taosMemoryFreeClear(pColInfoData->pData); } - blockDataDestroy(pBlock); + + taosArrayDestroy(pBlock->pDataBlock); + taosMemoryFreeClear(pBlock->pBlockAgg); } +static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); } + static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { int32_t tlen = 0; int32_t sz = 0; - tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + // tlen += taosEncodeFixedI64(buf, pRsp->consumerId); tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); if (pRsp->numOfTopics == 0) return tlen; - tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas); + tlen += tEncodeSSchemaWrapper(buf, pRsp->schema); if (pRsp->pBlockData) { sz = taosArrayGetSize(pRsp->pBlockData); } @@ -128,15 +141,15 @@ static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { int32_t sz; - buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + // buf = taosDecodeFixedI64(buf, &pRsp->consumerId); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); if (pRsp->numOfTopics == 0) return buf; - pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); - if (pRsp->schemas == NULL) return NULL; - buf = tDecodeSSchemaWrapper(buf, pRsp->schemas); + pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pRsp->schema == NULL) return NULL; + buf = tDecodeSSchemaWrapper(buf, pRsp->schema); buf = taosDecodeFixedI32(buf, &sz); pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); for (int32_t i = 0; i < sz; i++) { @@ -148,13 +161,13 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { } static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { - if (pRsp->schemas) { - if (pRsp->schemas->nCols) { - tfree(pRsp->schemas->pSchema); + if (pRsp->schema) { + if (pRsp->schema->nCols) { + taosMemoryFreeClear(pRsp->schema->pSchema); } - free(pRsp->schemas); + taosMemoryFree(pRsp->schema); } - taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); + taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner); pRsp->pBlockData = NULL; } @@ -166,10 +179,8 @@ typedef struct SColumn { int64_t dataBlockId; }; - union { - int16_t colId; - int16_t slotId; - }; + int16_t colId; + int16_t slotId; char name[TSDB_COL_NAME_LEN]; int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) @@ -196,7 +207,7 @@ typedef struct SGroupbyExpr { typedef struct SFunctParam { int32_t type; - SColumn *pCol; + SColumn* pCol; SVariant param; } SFunctParam; @@ -214,12 +225,12 @@ typedef struct SResSchame { typedef struct SExprBasicInfo { SResSchema resSchema; int16_t numOfParams; // argument value of each function - SFunctParam *pParam; + SFunctParam* pParam; } SExprBasicInfo; typedef struct SExprInfo { - struct SExprBasicInfo base; - struct tExprNode *pExpr; + struct SExprBasicInfo base; + struct tExprNode* pExpr; } SExprInfo; typedef struct SStateWindow { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 17e019d9775fb5714a95490db44937b3706736bc..20d743046f62c69349c0a27c6b2fda826ffcecff 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -104,6 +104,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); @@ -112,16 +113,16 @@ void colDataTrim(SColumnInfoData* pColumnInfoData); size_t blockDataGetNumOfCols(const SSDataBlock* pBlock); size_t blockDataGetNumOfRows(const SSDataBlock* pBlock); -int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); -int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, - int32_t pageSize); -SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount); - +int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, + int32_t pageSize); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf); +SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount); + size_t blockDataGetSize(const SSDataBlock* pBlock); -size_t blockDataGetRowSize(const SSDataBlock* pBlock); +size_t blockDataGetRowSize(SSDataBlock* pBlock); double blockDataGetSerialRowSize(const SSDataBlock* pBlock); size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock); @@ -132,11 +133,13 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -void blockDataClearup(SSDataBlock* pDataBlock); +void blockDataCleanup(SSDataBlock* pDataBlock); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); void* blockDataDestroy(SSDataBlock* pBlock); +void blockDebugShowData(const SArray* dataBlocks); + #ifdef __cplusplus } #endif diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 89a0e6b4dc5adf1659fc2a582b64cc355b2c9c89..698352f63652f9e57ae9d44493358a2b3f1cbc85 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -93,7 +93,7 @@ typedef struct { #define schemaFLen(s) ((s)->flen) #define schemaVLen(s) ((s)->vlen) #define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) tfree((s)) +#define tdFreeSchema(s) taosMemoryFreeClear((s)) STSchema *tdDupSchema(const STSchema *pSchema); int32_t tdEncodeSchema(void **buf, STSchema *pSchema); @@ -493,7 +493,7 @@ typedef struct { #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) tfree(r) +#define kvRowFree(r) taosMemoryFreeClear(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) #define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) #define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) @@ -593,7 +593,7 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; - SColIdx *pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); + SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); if (pColIdx == NULL) return -1; pBuilder->pColIdx = pColIdx; } @@ -608,7 +608,7 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t col while (tlen > pBuilder->alloc - pBuilder->size) { pBuilder->alloc *= 2; } - void *buf = realloc(pBuilder->buf, pBuilder->alloc); + void *buf = taosMemoryRealloc(pBuilder->buf, pBuilder->alloc); if (buf == NULL) return -1; pBuilder->buf = buf; } diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 342d78382ab4ac680eca259739c3061b15b94a22..90aac6edcdec34c4efee9abacb995c7d3827f9f3 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -51,6 +51,7 @@ extern int32_t tsCompatibleModel; extern bool tsEnableSlaveQuery; extern bool tsPrintAuth; extern int64_t tsTickPerDay[3]; +extern int32_t tsMultiProcess; // monitor extern bool tsEnableMonitor; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bd4cc52837e0d90f7adf1c7078eae2ccd70bc8bd..b7e0c7cba4cbba3fe97fd0921b21ebca3e314ae7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -23,6 +23,7 @@ #include "tencode.h" #include "thash.h" #include "tlist.h" +#include "tname.h" #include "trow.h" #include "tuuid.h" @@ -108,6 +109,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_FUNC, + TSDB_MGMT_TABLE_INDEX, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -182,6 +184,13 @@ typedef struct SField { int32_t bytes; } SField; +typedef struct SRetention { + int32_t freq; + int32_t keep; + int8_t freqUnit; + int8_t keepUnit; +} SRetention; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -192,21 +201,13 @@ typedef struct SEp { typedef struct { int32_t contLen; - union { - int32_t vgId; - int32_t streamTaskId; - }; + int32_t vgId; } SMsgHead; -typedef struct { - int32_t workerType; - int32_t streamTaskId; -} SStreamExecMsgHead; - // Submit message for one table typedef struct SSubmitBlk { int64_t uid; // table unique id - int32_t tid; // table id + int64_t suid; // stable id int32_t padding; // TODO just for padding here int32_t sversion; // data schema version int32_t dataLen; // data part length, not including the SSubmitBlk head @@ -231,12 +232,12 @@ typedef struct { } SSubmitBlkIter; typedef struct { - int32_t totalLen; - int32_t len; - void* pMsg; + int32_t totalLen; + int32_t len; + const void* pMsg; } SSubmitMsgIter; -int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter); +int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); @@ -267,11 +268,18 @@ typedef struct SSchema { typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; + float xFilesFactor; + int32_t aggregationMethod; + int32_t delay; + int32_t ttl; int32_t numOfColumns; int32_t numOfTags; - SArray* pColumns; - SArray* pTags; - char comment[TSDB_STB_COMMENT_LEN]; + int32_t numOfSmas; + int32_t commentLen; + SArray* pColumns; // array of SField + SArray* pTags; // array of SField + SArray* pSmas; // array of SField + char* comment; } SMCreateStbReq; int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); @@ -461,7 +469,9 @@ typedef struct { int32_t tz; // query client timezone char intervalUnit; char slidingUnit; - char offsetUnit; + char + offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. + int8_t precision; int64_t interval; int64_t sliding; int64_t offset; @@ -471,6 +481,10 @@ typedef struct { int32_t code; } SQueryTableRsp; +int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); + +int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t numOfVgroups; @@ -493,10 +507,13 @@ typedef struct { int8_t cacheLastRow; int8_t ignoreExist; int8_t streamMode; + int32_t numOfRetensions; + SArray* pRetensions; // SRetention } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); int32_t tDeserializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); +void tFreeSCreateDbReq(SCreateDbReq* pReq); typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -697,6 +714,7 @@ typedef struct { int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); int32_t tDeserializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); +void tFreeSStatusRsp(SStatusRsp* pRsp); typedef struct { int32_t reserved; @@ -740,11 +758,13 @@ typedef struct { int8_t selfIndex; int8_t streamMode; SReplica replicas[TSDB_MAX_REPLICA]; - + int32_t numOfRetensions; + SArray* pRetensions; // SRetention } SCreateVnodeReq, SAlterVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); +int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq); typedef struct { int32_t vgId; @@ -862,6 +882,7 @@ void tFreeSShowRsp(SShowRsp* pRsp); typedef struct { int32_t type; char db[TSDB_DB_FNAME_LEN]; + char tb[TSDB_TABLE_NAME_LEN]; int64_t showId; int8_t free; } SRetrieveTableReq; @@ -879,6 +900,17 @@ typedef struct { char data[]; } SRetrieveTableRsp; +typedef struct { + int64_t handle; + int64_t useconds; + int8_t completed; // all results are returned to client + int8_t precision; + int8_t compressed; + int32_t compLen; + int32_t numOfRows; + char data[]; +} SRetrieveMetaTableRsp; + typedef struct { char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port int32_t port; @@ -1093,7 +1125,6 @@ int32_t tDeserializeSSchedulerHbReq(void* buf, int32_t bufLen, SSchedulerHbReq* void tFreeSSchedulerHbReq(SSchedulerHbReq* pReq); typedef struct { - uint64_t seqId; SQueryNodeEpId epId; SArray* taskStatus; // SArray } SSchedulerHbRsp; @@ -1128,10 +1159,10 @@ typedef struct { typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; + char outputSTbName[TSDB_TABLE_FNAME_LEN]; int8_t igExists; char* sql; - char* physicalPlan; - char* logicalPlan; + char* ast; } SCMCreateStreamReq; typedef struct { @@ -1277,11 +1308,11 @@ typedef struct { } SMqRebSubscribe; static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { - SMqRebSubscribe* pRebSub = (SMqRebSubscribe*)calloc(1, sizeof(SMqRebSubscribe)); + SMqRebSubscribe* pRebSub = (SMqRebSubscribe*)taosMemoryCalloc(1, sizeof(SMqRebSubscribe)); if (pRebSub == NULL) { goto _err; } - pRebSub->key = key; + pRebSub->key = strdup(key); pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebSub->lostConsumers == NULL) { goto _err; @@ -1299,7 +1330,7 @@ _err: taosArrayDestroy(pRebSub->lostConsumers); taosArrayDestroy(pRebSub->removedConsumers); taosArrayDestroy(pRebSub->newConsumers); - tfree(pRebSub); + taosMemoryFreeClear(pRebSub); return NULL; } @@ -1344,33 +1375,54 @@ typedef struct { int64_t tuid; } SDDropTopicReq; +typedef struct { + float xFilesFactor; + int8_t delayUnit; + int8_t nFuncIds; + int32_t* pFuncIds; + int64_t delay; +} SRSmaParam; + typedef struct SVCreateTbReq { int64_t ver; // use a general definition + char* dbFName; char* name; uint32_t ttl; uint32_t keep; - uint8_t type; union { + uint8_t info; struct { - tb_uid_t suid; - uint32_t nCols; - SSchema* pSchema; - uint32_t nTagCols; - SSchema* pTagSchema; + uint8_t rollup : 1; // 1 means rollup sma + uint8_t type : 7; + }; + }; + union { + struct { + tb_uid_t suid; + uint32_t nCols; + SSchema* pSchema; + uint32_t nTagCols; + SSchema* pTagSchema; + col_id_t nBSmaCols; + col_id_t* pBSmaCols; + SRSmaParam* pRSmaParam; } stbCfg; struct { tb_uid_t suid; SKVRow pTag; } ctbCfg; struct { - uint32_t nCols; - SSchema* pSchema; + uint32_t nCols; + SSchema* pSchema; + col_id_t nBSmaCols; + col_id_t* pBSmaCols; + SRSmaParam* pRSmaParam; } ntbCfg; }; } SVCreateTbReq, SVUpdateTbReq; typedef struct { - int tmp; // TODO: to avoid compile error + int32_t code; } SVCreateTbRsp, SVUpdateTbRsp; int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); @@ -1381,12 +1433,15 @@ typedef struct { SArray* pArray; } SVCreateTbBatchReq; +int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); +void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); + typedef struct { - int tmp; // TODO: to avoid compile error + SArray* rspList; // SArray } SVCreateTbBatchRsp; -int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); -void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); +int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); typedef struct { int64_t ver; @@ -1576,7 +1631,7 @@ static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { void* pIter = taosHashIterate(info, NULL); while (pIter != NULL) { SKv* kv = (SKv*)pIter; - tfree(kv->value); + taosMemoryFreeClear(kv->value); pIter = taosHashIterate(info, pIter); } } @@ -1599,13 +1654,13 @@ static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { } else { taosArrayDestroy(req->reqs); } - free(pReq); + taosMemoryFree(pReq); } static FORCE_INLINE void tFreeClientKv(void* pKv) { SKv* kv = (SKv*)pKv; if (kv) { - tfree(kv->value); + taosMemoryFreeClear(kv->value); } } @@ -1632,7 +1687,7 @@ static FORCE_INLINE int32_t tEncodeSKv(SCoder* pEncoder, const SKv* pKv) { static FORCE_INLINE int32_t tDecodeSKv(SCoder* pDecoder, SKv* pKv) { if (tDecodeI32(pDecoder, &pKv->key) < 0) return -1; if (tDecodeI32(pDecoder, &pKv->valueLen) < 0) return -1; - pKv->value = malloc(pKv->valueLen + 1); + pKv->value = taosMemoryMalloc(pKv->valueLen + 1); if (pKv->value == NULL) return -1; if (tDecodeCStrTo(pDecoder, (char*)pKv->value) < 0) return -1; return 0; @@ -1886,7 +1941,7 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapp static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->nCols); - pSW->pSchema = (SSchema*)calloc(pSW->nCols, sizeof(SSchema)); + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { return NULL; } @@ -1896,12 +1951,47 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) } return buf; } + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + int8_t igExists; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; // strlen + 1 + int32_t sqlLen; // strlen + 1 + int32_t astLen; // strlen + 1 + char* expr; + char* tagsFilter; + char* sql; + char* ast; +} SMCreateSmaReq; + +int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); +int32_t tDeserializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); +void tFreeSMCreateSmaReq(SMCreateSmaReq* pReq); + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igNotExists; +} SMDropSmaReq; + +int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); +int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); + typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. char indexName[TSDB_INDEX_NAME_LEN]; - char timezone[TD_TIMEZONE_LEN]; // sma data expired if timezone changes. + char timezone[TD_TIMEZONE_LEN]; int32_t exprLen; int32_t tagsFilterLen; int64_t indexUid; @@ -1919,13 +2009,14 @@ typedef struct { } SVCreateTSmaReq; typedef struct { - int8_t type; // 0 status report, 1 update data - char indexName[TSDB_INDEX_NAME_LEN]; // - STimeWindow windows; + int8_t type; // 0 status report, 1 update data + int64_t indexUid; + int64_t skey; // start TS key of interval/sliding window } STSmaMsg; typedef struct { int64_t ver; // use a general definition + int64_t indexUid; char indexName[TSDB_INDEX_NAME_LEN]; } SVDropTSmaReq; @@ -1985,8 +2076,8 @@ typedef struct { static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { if (pSma) { - tfree(pSma->expr); - tfree(pSma->tagsFilter); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); } } @@ -1996,19 +2087,24 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { for (uint32_t i = 0; i < pSW->number; ++i) { tdDestroyTSma(pSW->tSma + i); } - tfree(pSW->tSma); + taosMemoryFreeClear(pSW->tSma); } } } +static FORCE_INLINE void tdFreeTSmaWrapper(STSmaWrapper* pSW) { + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); +} + static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSma->version); tlen += taosEncodeFixedI8(buf, pSma->intervalUnit); tlen += taosEncodeFixedI8(buf, pSma->slidingUnit); + tlen += taosEncodeFixedI8(buf, pSma->timezoneInt); tlen += taosEncodeString(buf, pSma->indexName); - tlen += taosEncodeString(buf, pSma->timezone); tlen += taosEncodeFixedI32(buf, pSma->exprLen); tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen); tlen += taosEncodeFixedI64(buf, pSma->indexUid); @@ -2042,8 +2138,8 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { buf = taosDecodeFixedI8(buf, &pSma->version); buf = taosDecodeFixedI8(buf, &pSma->intervalUnit); buf = taosDecodeFixedI8(buf, &pSma->slidingUnit); + buf = taosDecodeFixedI8(buf, &pSma->timezoneInt); buf = taosDecodeStringTo(buf, pSma->indexName); - buf = taosDecodeStringTo(buf, pSma->timezone); buf = taosDecodeFixedI32(buf, &pSma->exprLen); buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen); buf = taosDecodeFixedI64(buf, &pSma->indexUid); @@ -2076,7 +2172,7 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->number); - pSW->tSma = (STSma*)calloc(pSW->number, sizeof(STSma)); + pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma)); if (pSW->tSma == NULL) { return NULL; } @@ -2086,7 +2182,7 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { for (uint32_t j = i; j >= 0; --i) { tdDestroyTSma(pSW->tSma + j); } - free(pSW->tSma); + taosMemoryFree(pSW->tSma); return NULL; } } @@ -2114,25 +2210,16 @@ typedef struct { int8_t mqMsgType; int32_t code; int32_t epoch; + int64_t consumerId; } SMqRspHead; -typedef struct { - int64_t consumerId; - SSchemaWrapper* schemas; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t numOfTopics; - SArray* pBlockData; // SArray -} SMqPollRsp; - -// one req for one vg+topic typedef struct { SMsgHead head; int64_t consumerId; int64_t blockingTime; int32_t epoch; + int8_t withSchema; char cgroup[TSDB_CGROUP_LEN]; int64_t currentOffset; @@ -2151,19 +2238,22 @@ typedef struct { } SMqSubTopicEp; typedef struct { - int64_t consumerId; - char cgroup[TSDB_CGROUP_LEN]; - SArray* topics; // SArray -} SMqCMGetSubEpRsp; + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + // TODO: replace with topic name + int32_t numOfTopics; + // TODO: remove from msg + SSchemaWrapper* schema; + SArray* pBlockData; // SArray +} SMqPollRsp; typedef struct { SMqRspHead head; - union { - SMqPollRsp consumeRsp; - SMqCMGetSubEpRsp getEpRsp; - }; - void* extra; -} SMqMsgWrapper; + char cgroup[TSDB_CGROUP_LEN]; + SArray* topics; // SArray +} SMqCMGetSubEpRsp; typedef struct { int32_t curBlock; @@ -2171,11 +2261,13 @@ typedef struct { void** uData; } SMqRowIter; -struct tmq_message_t_v1 { - SMqPollRsp rsp; +struct tmq_message_t { + SMqPollRsp msg; + void* vg; SMqRowIter iter; }; +#if 0 struct tmq_message_t { SMqRspHead head; union { @@ -2187,6 +2279,7 @@ struct tmq_message_t { int32_t curRow; void** uData; }; +#endif static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } @@ -2239,8 +2332,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeString(buf, pRsp->cgroup); + // tlen += taosEncodeString(buf, pRsp->cgroup); int32_t sz = taosArrayGetSize(pRsp->topics); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { @@ -2251,8 +2343,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu } static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeStringTo(buf, pRsp->cgroup); + // buf = taosDecodeStringTo(buf, pRsp->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -2266,56 +2357,6 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } return buf; } - -enum { - STREAM_TASK_STATUS__RUNNING = 1, - STREAM_TASK_STATUS__STOP, -}; - -typedef struct { - int64_t streamId; - int32_t taskId; - int32_t level; - int8_t status; - char* qmsg; - void* executor; - // void* stateStore; - // storage handle -} SStreamTask; - -static FORCE_INLINE SStreamTask* streamTaskNew(int64_t streamId, int32_t level) { - SStreamTask* pTask = (SStreamTask*)calloc(1, sizeof(SStreamTask)); - if (pTask == NULL) { - return NULL; - } - pTask->taskId = tGenIdPI32(); - pTask->status = STREAM_TASK_STATUS__RUNNING; - pTask->qmsg = NULL; - return pTask; -} - -int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask); -int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask); -void tFreeSStreamTask(SStreamTask* pTask); - -typedef struct { - SMsgHead head; - SStreamTask* task; -} SStreamTaskDeployReq; - -typedef struct { - int32_t reserved; -} SStreamTaskDeployRsp; - -typedef struct { - SStreamExecMsgHead head; - // TODO: other info needed by task -} SStreamTaskExecReq; - -typedef struct { - int32_t reserved; -} SStreamTaskExecRsp; - #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h new file mode 100644 index 0000000000000000000000000000000000000000..1b3311b40095a461796973887a38d40b2e26b463 --- /dev/null +++ b/include/common/tmsgcb.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_MSG_CB_H_ +#define _TD_COMMON_MSG_CB_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SRpcMsg SRpcMsg; +typedef struct SEpSet SEpSet; +typedef struct SMgmtWrapper SMgmtWrapper; +typedef enum { + QUERY_QUEUE, + FETCH_QUEUE, + READ_QUEUE, + WRITE_QUEUE, + APPLY_QUEUE, + SYNC_QUEUE, + MERGE_QUEUE, + QUEUE_MAX, +} EQueueType; + +typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); +typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype); +typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, SEpSet* epSet, SRpcMsg* pReq); +typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); +typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, SRpcMsg* pRsp); + +typedef struct { + SMgmtWrapper* pWrapper; + PutToQueueFp queueFps[QUEUE_MAX]; + GetQueueSizeFp qsizeFp; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; +} SMsgCb; + +int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq); +int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); +int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq); +int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq); +void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_MSG_CB_H_*/ diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a596794b3d9ac493f775bd2d6f6bb74f12075036..051ee34644ce214d5be67109944233a868b66a58 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -127,6 +127,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SMA, "mnode-create-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_SMA, "mnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "mnode-qnode-list", NULL, NULL) @@ -190,6 +192,11 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_PIPE_EXEC, "vnode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_MERGE_EXEC, "vnode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_WRITE_EXEC, "vnode-task-write-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) @@ -202,7 +209,13 @@ enum { TD_NEW_MSG_SEG(TDMT_SND_MSG) TD_DEF_MSG_TYPE(TDMT_SND_TASK_DEPLOY, "snode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_SND_TASK_EXEC, "snode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_SND_TASK_PIPE_EXEC, "snode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_SND_TASK_MERGE_EXEC, "snode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + // Requests handled by SCHEDULER + TD_NEW_MSG_SEG(TDMT_SCH_MSG) + TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "scheduler-link-broken", NULL, NULL) + #if defined(TD_MSG_NUMBER_) TDMT_MAX #endif diff --git a/include/common/tname.h b/include/common/tname.h index 6de38a68ee9ac782eb52e02f097d24633331ea13..ffa4f8f253a57c5d9897a7cc0f1306998b2857a5 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -17,7 +17,6 @@ #define _TD_COMMON_NAME_H_ #include "tdef.h" -#include "tmsg.h" #ifdef __cplusplus extern "C" { @@ -61,7 +60,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type); int32_t tNameSetAcctId(SName* dst, int32_t acctId); -SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); +bool tNameDBNameEqual(SName* left, SName* right); + #ifdef __cplusplus } diff --git a/include/common/trow.h b/include/common/trow.h index 47edf6f1ad45b2181ef75f25d78929ceafb58be8..fc99cbc5b29506849d5a5ca272d83a79042f7739 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -166,6 +166,7 @@ typedef struct { #define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) +#define TD_ROW_INFO(r) ((r)->info) #define TD_ROW_TYPE(r) ((r)->type) #define TD_ROW_DELETE(r) ((r)->del) #define TD_ROW_ENDIAN(r) ((r)->endian) @@ -180,6 +181,7 @@ typedef struct { // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN) +#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i)) #define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) #define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) #define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) @@ -473,6 +475,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } + TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); uint32_t len = 0; @@ -968,10 +971,14 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in #endif return TSDB_CODE_SUCCESS; } - if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) { + + if (TD_COL_ROWS_NORM(pCol)) { + pVal->valType = TD_VTYPE_NORM; + } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) { return terrno; } - if (TD_COL_ROWS_NORM(pCol) || tdValTypeIsNorm(pVal->valType)) { + + if (tdValTypeIsNorm(pVal->valType)) { if (IS_VAR_DATA_TYPE(pCol->type)) { pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); } else { diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 27d509e931d8197b5fab120241dc2afd5b606087..da34ca62536872599cb0e91db172d30a83e73346 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -51,130 +51,133 @@ #define TK_USER 33 #define TK_PRIVILEGE 34 #define TK_DROP 35 -#define TK_SHOW 36 -#define TK_DNODE 37 -#define TK_PORT 38 -#define TK_NK_INTEGER 39 -#define TK_DNODES 40 -#define TK_NK_IPTOKEN 41 -#define TK_LOCAL 42 -#define TK_QNODE 43 -#define TK_ON 44 -#define TK_QNODES 45 -#define TK_DATABASE 46 -#define TK_DATABASES 47 -#define TK_USE 48 -#define TK_IF 49 -#define TK_NOT 50 -#define TK_EXISTS 51 -#define TK_BLOCKS 52 -#define TK_CACHE 53 -#define TK_CACHELAST 54 -#define TK_COMP 55 -#define TK_DAYS 56 -#define TK_FSYNC 57 -#define TK_MAXROWS 58 -#define TK_MINROWS 59 -#define TK_KEEP 60 -#define TK_PRECISION 61 -#define TK_QUORUM 62 -#define TK_REPLICA 63 -#define TK_TTL 64 -#define TK_WAL 65 -#define TK_VGROUPS 66 -#define TK_SINGLE_STABLE 67 -#define TK_STREAM_MODE 68 -#define TK_RETENTIONS 69 -#define TK_FILE_FACTOR 70 -#define TK_NK_FLOAT 71 -#define TK_TABLE 72 -#define TK_NK_LP 73 -#define TK_NK_RP 74 -#define TK_STABLE 75 -#define TK_TABLES 76 -#define TK_STABLES 77 -#define TK_ADD 78 -#define TK_COLUMN 79 -#define TK_MODIFY 80 -#define TK_RENAME 81 -#define TK_TAG 82 -#define TK_SET 83 -#define TK_NK_EQ 84 -#define TK_USING 85 -#define TK_TAGS 86 -#define TK_NK_DOT 87 -#define TK_NK_COMMA 88 -#define TK_COMMENT 89 -#define TK_BOOL 90 -#define TK_TINYINT 91 -#define TK_SMALLINT 92 -#define TK_INT 93 -#define TK_INTEGER 94 -#define TK_BIGINT 95 -#define TK_FLOAT 96 -#define TK_DOUBLE 97 -#define TK_BINARY 98 -#define TK_TIMESTAMP 99 -#define TK_NCHAR 100 -#define TK_UNSIGNED 101 -#define TK_JSON 102 -#define TK_VARCHAR 103 -#define TK_MEDIUMBLOB 104 -#define TK_BLOB 105 -#define TK_VARBINARY 106 -#define TK_DECIMAL 107 -#define TK_SMA 108 -#define TK_ROLLUP 109 -#define TK_INDEX 110 -#define TK_FULLTEXT 111 -#define TK_FUNCTION 112 -#define TK_INTERVAL 113 -#define TK_TOPIC 114 -#define TK_AS 115 -#define TK_MNODES 116 -#define TK_NK_BOOL 117 -#define TK_NK_VARIABLE 118 -#define TK_BETWEEN 119 -#define TK_IS 120 -#define TK_NULL 121 -#define TK_NK_LT 122 -#define TK_NK_GT 123 -#define TK_NK_LE 124 -#define TK_NK_GE 125 -#define TK_NK_NE 126 -#define TK_LIKE 127 -#define TK_MATCH 128 -#define TK_NMATCH 129 -#define TK_IN 130 -#define TK_FROM 131 -#define TK_JOIN 132 -#define TK_INNER 133 -#define TK_SELECT 134 -#define TK_DISTINCT 135 -#define TK_WHERE 136 -#define TK_PARTITION 137 -#define TK_BY 138 -#define TK_SESSION 139 -#define TK_STATE_WINDOW 140 -#define TK_SLIDING 141 -#define TK_FILL 142 -#define TK_VALUE 143 -#define TK_NONE 144 -#define TK_PREV 145 -#define TK_LINEAR 146 -#define TK_NEXT 147 -#define TK_GROUP 148 -#define TK_HAVING 149 -#define TK_ORDER 150 -#define TK_SLIMIT 151 -#define TK_SOFFSET 152 -#define TK_LIMIT 153 -#define TK_OFFSET 154 -#define TK_ASC 155 -#define TK_DESC 156 -#define TK_NULLS 157 -#define TK_FIRST 158 -#define TK_LAST 159 +#define TK_DNODE 36 +#define TK_PORT 37 +#define TK_NK_INTEGER 38 +#define TK_DNODES 39 +#define TK_NK_IPTOKEN 40 +#define TK_LOCAL 41 +#define TK_QNODE 42 +#define TK_ON 43 +#define TK_DATABASE 44 +#define TK_USE 45 +#define TK_IF 46 +#define TK_NOT 47 +#define TK_EXISTS 48 +#define TK_BLOCKS 49 +#define TK_CACHE 50 +#define TK_CACHELAST 51 +#define TK_COMP 52 +#define TK_DAYS 53 +#define TK_FSYNC 54 +#define TK_MAXROWS 55 +#define TK_MINROWS 56 +#define TK_KEEP 57 +#define TK_PRECISION 58 +#define TK_QUORUM 59 +#define TK_REPLICA 60 +#define TK_TTL 61 +#define TK_WAL 62 +#define TK_VGROUPS 63 +#define TK_SINGLE_STABLE 64 +#define TK_STREAM_MODE 65 +#define TK_RETENTIONS 66 +#define TK_FILE_FACTOR 67 +#define TK_NK_FLOAT 68 +#define TK_TABLE 69 +#define TK_NK_LP 70 +#define TK_NK_RP 71 +#define TK_STABLE 72 +#define TK_ADD 73 +#define TK_COLUMN 74 +#define TK_MODIFY 75 +#define TK_RENAME 76 +#define TK_TAG 77 +#define TK_SET 78 +#define TK_NK_EQ 79 +#define TK_USING 80 +#define TK_TAGS 81 +#define TK_NK_DOT 82 +#define TK_NK_COMMA 83 +#define TK_COMMENT 84 +#define TK_BOOL 85 +#define TK_TINYINT 86 +#define TK_SMALLINT 87 +#define TK_INT 88 +#define TK_INTEGER 89 +#define TK_BIGINT 90 +#define TK_FLOAT 91 +#define TK_DOUBLE 92 +#define TK_BINARY 93 +#define TK_TIMESTAMP 94 +#define TK_NCHAR 95 +#define TK_UNSIGNED 96 +#define TK_JSON 97 +#define TK_VARCHAR 98 +#define TK_MEDIUMBLOB 99 +#define TK_BLOB 100 +#define TK_VARBINARY 101 +#define TK_DECIMAL 102 +#define TK_SMA 103 +#define TK_ROLLUP 104 +#define TK_SHOW 105 +#define TK_DATABASES 106 +#define TK_TABLES 107 +#define TK_STABLES 108 +#define TK_MNODES 109 +#define TK_MODULES 110 +#define TK_QNODES 111 +#define TK_FUNCTIONS 112 +#define TK_INDEXES 113 +#define TK_FROM 114 +#define TK_LIKE 115 +#define TK_INDEX 116 +#define TK_FULLTEXT 117 +#define TK_FUNCTION 118 +#define TK_INTERVAL 119 +#define TK_TOPIC 120 +#define TK_AS 121 +#define TK_NK_BOOL 122 +#define TK_NK_VARIABLE 123 +#define TK_BETWEEN 124 +#define TK_IS 125 +#define TK_NULL 126 +#define TK_NK_LT 127 +#define TK_NK_GT 128 +#define TK_NK_LE 129 +#define TK_NK_GE 130 +#define TK_NK_NE 131 +#define TK_MATCH 132 +#define TK_NMATCH 133 +#define TK_IN 134 +#define TK_JOIN 135 +#define TK_INNER 136 +#define TK_SELECT 137 +#define TK_DISTINCT 138 +#define TK_WHERE 139 +#define TK_PARTITION 140 +#define TK_BY 141 +#define TK_SESSION 142 +#define TK_STATE_WINDOW 143 +#define TK_SLIDING 144 +#define TK_FILL 145 +#define TK_VALUE 146 +#define TK_NONE 147 +#define TK_PREV 148 +#define TK_LINEAR 149 +#define TK_NEXT 150 +#define TK_GROUP 151 +#define TK_HAVING 152 +#define TK_ORDER 153 +#define TK_SLIMIT 154 +#define TK_SOFFSET 155 +#define TK_LIMIT 156 +#define TK_OFFSET 157 +#define TK_ASC 158 +#define TK_DESC 159 +#define TK_NULLS 160 +#define TK_FIRST 161 +#define TK_LAST 162 #define TK_NK_SPACE 300 #define TK_NK_COMMENT 301 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index a936ea3b5410a4942bd8775a169e7841bc0083cd..59af14c22647a7013bcbf4117c9e737d902b3418 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -46,7 +46,7 @@ typedef struct { #define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) @@ -159,6 +159,7 @@ typedef struct { (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL)) +// TODO remove this function static FORCE_INLINE bool isNull(const void *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 7f7b8a8d53f1fa249704d6e5510273ae4247da4a..528cea8828a8cb63356ed04cb6514df876feb772 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -16,29 +16,20 @@ #ifndef _TD_BNODE_H_ #define _TD_BNODE_H_ +#include "tmsgcb.h" + #ifdef __cplusplus extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; typedef struct SBnode SBnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { - int64_t numOfErrors; } SBnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SMsgCb msgCb; } SBnodeOpt; /* ------------------------ SBnode ------------------------ */ @@ -76,13 +67,6 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad); */ int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs); -/** - * @brief Drop a bnode. - * - * @param path Path of the bnode. - */ -void bndDestroy(const char *path); - #ifdef __cplusplus } #endif diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 068e9a69170bfd1659b2961bef220b765d74c35e..8d19ce23dfdfce22a3e0e9b0961f58aa9319bc74 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -33,8 +33,7 @@ typedef struct SDnode SDnode; int32_t dndInit(); /** - * @brief clear the environment - * + * @brief Clear the environment */ void dndCleanup(); @@ -42,22 +41,24 @@ void dndCleanup(); typedef struct { int32_t numOfSupportVnodes; uint16_t serverPort; - char dataDir[TSDB_FILENAME_LEN]; + char dataDir[PATH_MAX]; char localEp[TSDB_EP_LEN]; char localFqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; char secondEp[TSDB_EP_LEN]; SDiskCfg *pDisks; int32_t numOfDisks; -} SDnodeObjCfg; +} SDnodeOpt; + +typedef enum { DND_EVENT_START, DND_EVENT_STOP = 1, DND_EVENT_RELOAD } EDndEvent; /** * @brief Initialize and start the dnode. * - * @param pCfg Config of the dnode. + * @param pOption Option of the dnode. * @return SDnode* The dnode object. */ -SDnode *dndCreate(SDnodeObjCfg *pCfg); +SDnode *dndCreate(const SDnodeOpt *pOption); /** * @brief Stop and cleanup the dnode. @@ -66,6 +67,21 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg); */ void dndClose(SDnode *pDnode); +/** + * @brief Run dnode until specific event is receive. + * + * @param pDnode The dnode object to run. + */ +int32_t dndRun(SDnode *pDnode); + +/** + * @brief Handle event in the dnode. + * + * @param pDnode The dnode object to close. + * @param event The event to handle. + */ +void dndHandleEvent(SDnode *pDnode, EDndEvent event); + #ifdef __cplusplus } #endif diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index d295e772e898e62bce7e9b822309a79640368a0d..5b88a9d6aff34bf3a151650f42c48e4ef325e81d 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -17,33 +17,24 @@ #define _TD_MND_H_ #include "monitor.h" +#include "tmsg.h" +#include "tmsgcb.h" +#include "trpc.h" #ifdef __cplusplus extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SMnode SMnode; -typedef struct SMnodeMsg SMnodeMsg; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef struct SMnode SMnode; typedef struct { - int32_t dnodeId; - int64_t clusterId; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; - SDnode *pDnode; - PutReqToMWriteQFp putReqToMWriteQFp; - PutReqToMReadQFp putReqToMReadQFp; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; + SMsgCb msgCb; } SMnodeOpt; /* ------------------------ SMnode ------------------------ */ @@ -73,11 +64,11 @@ void mndClose(SMnode *pMnode); int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption); /** - * @brief Drop a mnode. + * @brief Start mnode * - * @param path Path of the mnode. + * @param pMnode The mnode object. */ -void mndDestroy(const char *path); +int32_t mndStart(SMnode *pMnode); /** * @brief Get mnode monitor info. @@ -104,37 +95,13 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr */ int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); -/** - * @brief Initialize mnode msg. - * - * @param pMnode The mnode object. - * @param pMsg The request rpc msg. - * @return int32_t The created mnode msg. - */ -SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg); - -/** - * @brief Cleanup mnode msg. - * - * @param pMsg The request msg. - */ -void mndCleanupMsg(SMnodeMsg *pMsg); - -/** - * @brief Cleanup mnode msg. - * - * @param pMsg The request msg. - * @param code The error code. - */ -void mndSendRsp(SMnodeMsg *pMsg, int32_t code); - /** * @brief Process the read, write, sync request. * * @param pMsg The request msg. * @return int32_t 0 for success, -1 for failure. */ -void mndProcessMsg(SMnodeMsg *pMsg); +int32_t mndProcessMsg(SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index d04e9f817e7b601a22212765ba6612bb6ec649f2..a28c093e85bfbb7c5e1c3789e71b23e700b0549c 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -119,10 +119,11 @@ typedef enum { SDB_CONSUMER = 13, SDB_TOPIC = 14, SDB_VGROUP = 15, - SDB_STB = 16, - SDB_DB = 17, - SDB_FUNC = 18, - SDB_MAX = 19 + SDB_SMA = 16, + SDB_STB = 17, + SDB_DB = 18, + SDB_FUNC = 19, + SDB_MAX = 20 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 3de2986047c4b5bf1e047e294b46999149abd433..89553f978beaacd7f54cdce5ab786d510ecd9fc7 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -16,16 +16,14 @@ #ifndef _TD_QNODE_H_ #define _TD_QNODE_H_ +#include "tmsgcb.h" + #ifdef __cplusplus extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; typedef struct SQnode SQnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { int64_t numOfStartTask; @@ -39,13 +37,7 @@ typedef struct { } SQnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SMsgCb msgCb; } SQnodeOpt; /* ------------------------ SQnode ------------------------ */ @@ -78,10 +70,9 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad); * * @param pQnode The qnode object. * @param pMsg The request message - * @param pRsp The response message - * @return int32_t 0 for success, -1 for failure */ -int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg); +int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index b25f8a8666bec1e76ef394572ae24f9fba0e14dd..611bff49f116c3a10363c607474ebb90eb275186 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -16,7 +16,7 @@ #ifndef _TD_SNODE_H_ #define _TD_SNODE_H_ -#include "tcommon.h" +#include "tmsgcb.h" #include "tmsg.h" #include "trpc.h" @@ -25,24 +25,14 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; typedef struct SSnode SSnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { - int64_t numOfErrors; + int32_t reserved; } SSnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SMsgCb msgCb; } SSnodeOpt; /* ------------------------ SSnode ------------------------ */ @@ -77,20 +67,9 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad); * @param pSnode The snode object. * @param pMsg The request message * @param pRsp The response message - * @return int32_t 0 for success, -1 for failure */ -// int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); - -int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); - -int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg); - -/** - * @brief Drop a snode. - * - * @param path Path of the snode. - */ -void sndDestroy(const char *path); +void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); +void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index f217277b805dd0fefdd1746daf499d1d6a6db680..9f0d4b11c23d0d313a452033503c27b76f69c3e6 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -103,16 +103,17 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) * @param pDBName (input, full db name) - * @param forceUpdate (input, force update db vgroup info from mnode) * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ -int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList); +int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, SArray** pVgroupList); int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); +int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName); + int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); /** @@ -120,7 +121,7 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) - * @param pTableName (input, table name, NOT including db name) + * @param pTableName (input, table name) * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ @@ -131,7 +132,7 @@ int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSe * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) - * @param pTableName (input, table name, NOT including db name) + * @param pTableName (input, table name) * @param pTableMeta(output, table meta data, NEED to free it by calller) * @return error code */ @@ -140,28 +141,38 @@ int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpS int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); +/** + * Force refresh DB's local cached vgroup info. + * @param pCtg (input, got with catalogGetHandle) + * @param pTrans (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param dbFName (input, db full name) + * @return error code + */ +int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName); + /** * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) - * @param pTableName (input, table name, NOT including db name) + * @param pTableName (input, table name) * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ - int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); +int32_t catalogRefreshTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); /** * Force refresh a table's local cached meta data and get the new one. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) - * @param pTableName (input, table name, NOT including db name) + * @param pTableName (input, table name) * @param pTableMeta(output, table meta data, NEED to free it by calller) * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ - int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); +int32_t catalogRefreshGetTableMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); @@ -170,7 +181,7 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) - * @param pTableName (input, table name, NOT including db name) + * @param pTableName (input, table name) * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @return error code */ @@ -181,7 +192,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) - * @param pTableName (input, table name, NOT including db name) + * @param pTableName (input, table name) * @param vgInfo (output, vgroup info) * @return error code */ diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index d4af51fc21c72084de78ea6c1872a7065922e1e5..4ad7e2dfc2c7b5ce8065534c315e800c0d5eec26 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "tcommon.h" +#include "query.h" typedef void* qTaskInfo_t; typedef void* DataSinkHandle; @@ -30,23 +31,28 @@ struct SSubplan; typedef struct SReadHandle { void* reader; void* meta; + void* config; } SReadHandle; - /** - * Create the exec task for streaming mode - * @param pMsg - * @param streamReadHandle - * @return - */ -qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); +#define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1 +#define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2 + +/** + * Create the exec task for streaming mode + * @param pMsg + * @param streamReadHandle + * @return + */ +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle); /** * Set the input data block for the stream scan. * @param tinfo * @param input + * @param type * @return */ -int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type); /** * Update the table id list, add or remove. @@ -58,16 +64,17 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); */ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); - /** - * Create the exec task object according to task json - * @param readHandle - * @param vgId - * @param pTaskInfoMsg - * @param pTaskInfo - * @param qId - * @return - */ -int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); +/** + * Create the exec task object according to task json + * @param readHandle + * @param vgId + * @param pTaskInfoMsg + * @param pTaskInfo + * @param qId + * @return + */ +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); /** * The main task execution function, including query on both table and multiple tables, @@ -77,7 +84,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, * @param handle * @return */ -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds); +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -140,7 +147,8 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t * @param numOfIndex * @return */ -//int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); +// int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* +// groupByIndex, int32_t numOfIndex); /** * Update the table id list of a given query. @@ -163,19 +171,19 @@ void* qOpenTaskMgmt(int32_t vgId); * broadcast the close information and wait for all query stop. * @param pExecutor */ -void qTaskMgmtNotifyClosing(void* pExecutor); +void qTaskMgmtNotifyClosing(void* pExecutor); /** * Re-open the query handle management module when opening the vnode again. * @param pExecutor */ -void qQueryMgmtReOpen(void *pExecutor); +void qQueryMgmtReOpen(void* pExecutor); /** * Close query mgmt and clean up resources. * @param pExecutor */ -void qCleanupTaskMgmt(void* pExecutor); +void qCleanupTaskMgmt(void* pExecutor); /** * Add the query into the query mgmt object @@ -184,7 +192,7 @@ void qCleanupTaskMgmt(void* pExecutor); * @param qInfo * @return */ -void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo); +void** qRegisterTask(void* pMgmt, uint64_t qId, void* qInfo); /** * acquire the query handle according to the key from query mgmt object. diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 1abdca465e9ff1185d93088e579d7e886888724a..6c4774c3be60249960b6e633c705fa82d553d199 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -163,7 +163,7 @@ typedef struct SInputColumnInfoData { typedef struct SqlFunctionCtx { SInputColumnInfoData input; SResultDataInfo resDataInfo; - uint32_t order; // asc|desc + uint32_t order; // data block scanner order: asc|desc //////////////////////////////////////////////////////////////// int32_t startRow; // start row index int32_t size; // handled processed row number @@ -327,7 +327,7 @@ bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - struct SFillColInfo* pFillCol, void* handle); + struct SFillColInfo* pFillCol, const char* id); void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 35832fa298b2328b74a7836338957b1ae7e96471..85a9cd0b23ef7ec5befce7db192f9acc9be61d19 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -57,18 +57,19 @@ typedef enum EFunctionType { // math function FUNCTION_TYPE_ABS = 1000, - FUNCTION_TYPE_ACOS, - FUNCTION_TYPE_ASION, - FUNCTION_TYPE_ATAN, - FUNCTION_TYPE_CEIL, - FUNCTION_TYPE_COS, - FUNCTION_TYPE_FLOOR, FUNCTION_TYPE_LOG, FUNCTION_TYPE_POW, + FUNCTION_TYPE_SQRT, + FUNCTION_TYPE_CEIL, + FUNCTION_TYPE_FLOOR, FUNCTION_TYPE_ROUND, + FUNCTION_TYPE_SIN, - FUNCTION_TYPE_SQRT, + FUNCTION_TYPE_COS, FUNCTION_TYPE_TAN, + FUNCTION_TYPE_ASIN, + FUNCTION_TYPE_ACOS, + FUNCTION_TYPE_ATAN, // string function FUNCTION_TYPE_CHAR_LENGTH = 1500, diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 5dbfd6c2c277f40e1d5a79a6037e6e8f4283bc29..48a3a9bda8b200cab6c46520be64b017fcd98c00 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -179,7 +179,8 @@ typedef struct SAlterDnodeStmt { typedef struct SShowStmt { ENodeType type; - char dbName[TSDB_DB_NAME_LEN]; + SNode* pDbName; // SValueNode + SNode* pTbNamePattern; // SValueNode } SShowStmt; typedef enum EIndexType { @@ -198,6 +199,7 @@ typedef struct SIndexOptions { typedef struct SCreateIndexStmt { ENodeType type; EIndexType indexType; + bool ignoreExists; char indexName[TSDB_INDEX_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; SNodeList* pCols; @@ -206,6 +208,7 @@ typedef struct SCreateIndexStmt { typedef struct SDropIndexStmt { ENodeType type; + bool ignoreNotExists; char indexName[TSDB_INDEX_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; } SDropIndexStmt; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index d7e67be9e03868f57737af52d80544f2af410696..4c83a30bb9c5f55ffb3bf2ebfba2442019a19fd3 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -78,7 +78,6 @@ typedef enum ENodeType { QUERY_NODE_CREATE_DATABASE_STMT, QUERY_NODE_DROP_DATABASE_STMT, QUERY_NODE_ALTER_DATABASE_STMT, - QUERY_NODE_SHOW_DATABASES_STMT, // temp QUERY_NODE_CREATE_TABLE_STMT, QUERY_NODE_CREATE_SUBTABLE_CLAUSE, QUERY_NODE_CREATE_MULTI_TABLE_STMT, @@ -86,20 +85,13 @@ typedef enum ENodeType { QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT, QUERY_NODE_ALTER_TABLE_STMT, - QUERY_NODE_SHOW_TABLES_STMT, // temp - QUERY_NODE_SHOW_STABLES_STMT, QUERY_NODE_CREATE_USER_STMT, QUERY_NODE_ALTER_USER_STMT, QUERY_NODE_DROP_USER_STMT, - QUERY_NODE_SHOW_USERS_STMT, QUERY_NODE_USE_DATABASE_STMT, QUERY_NODE_CREATE_DNODE_STMT, QUERY_NODE_DROP_DNODE_STMT, QUERY_NODE_ALTER_DNODE_STMT, - QUERY_NODE_SHOW_DNODES_STMT, - QUERY_NODE_SHOW_VGROUPS_STMT, - QUERY_NODE_SHOW_MNODES_STMT, - QUERY_NODE_SHOW_QNODES_STMT, QUERY_NODE_CREATE_INDEX_STMT, QUERY_NODE_DROP_INDEX_STMT, QUERY_NODE_CREATE_QNODE_STMT, @@ -107,6 +99,18 @@ typedef enum ENodeType { QUERY_NODE_CREATE_TOPIC_STMT, QUERY_NODE_DROP_TOPIC_STMT, QUERY_NODE_ALTER_LOCAL_STMT, + QUERY_NODE_SHOW_DATABASES_STMT, + QUERY_NODE_SHOW_TABLES_STMT, + QUERY_NODE_SHOW_STABLES_STMT, + QUERY_NODE_SHOW_USERS_STMT, + QUERY_NODE_SHOW_DNODES_STMT, + QUERY_NODE_SHOW_VGROUPS_STMT, + QUERY_NODE_SHOW_MNODES_STMT, + QUERY_NODE_SHOW_MODULES_STMT, + QUERY_NODE_SHOW_QNODES_STMT, + QUERY_NODE_SHOW_FUNCTIONS_STMT, + QUERY_NODE_SHOW_INDEXES_STMT, + QUERY_NODE_SHOW_STREAMS_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN, @@ -131,6 +135,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN_INSERT, QUERY_NODE_PHYSICAL_SUBPLAN, @@ -165,6 +170,7 @@ void nodesDestroyNode(SNodeptr pNode); SNodeList* nodesMakeList(); int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); +int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 18417cb6080ec76856d2228a36f72cefdcb0d83f..4d025eb9b7b4d1608d99e442126c46fc2df9ea81 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -35,7 +35,7 @@ typedef struct SLogicNode { typedef enum EScanType { SCAN_TYPE_TAG, SCAN_TYPE_TABLE, - SCAN_TYPE_STABLE, + SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType; @@ -48,6 +48,7 @@ typedef struct SScanLogicNode { uint8_t scanFlag; // denotes reversed scan of data or not STimeWindow scanRange; SName tableName; + bool showRewrite; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -95,6 +96,7 @@ typedef struct SWindowLogicNode { int8_t intervalUnit; int8_t slidingUnit; SFillNode* pFill; + int64_t sessionGap; } SWindowLogicNode; typedef enum ESubplanType { @@ -110,7 +112,7 @@ typedef struct SSubplanId { int32_t subplanId; } SSubplanId; -typedef struct SSubLogicPlan { +typedef struct SLogicSubplan { ENodeType type; SSubplanId id; SNodeList* pChildren; @@ -120,7 +122,7 @@ typedef struct SSubLogicPlan { SVgroupsInfo* pVgroupList; int32_t level; int32_t splitFlag; -} SSubLogicPlan; +} SLogicSubplan; typedef struct SQueryLogicPlan { ENodeType type; @@ -154,7 +156,7 @@ typedef struct SPhysiNode { } SPhysiNode; typedef struct SScanPhysiNode { - SPhysiNode node; + SPhysiNode node; SNodeList* pScanCols; uint64_t uid; // unique id of the table int8_t tableType; @@ -164,10 +166,16 @@ typedef struct SScanPhysiNode { SName tableName; } SScanPhysiNode; -typedef SScanPhysiNode SSystemTableScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode SStreamScanPhysiNode; +typedef struct SSystemTableScanPhysiNode { + SScanPhysiNode scan; + SEpSet mgmtEpSet; + bool showRewrite; + int32_t accountId; +} SSystemTableScanPhysiNode; + typedef struct STableScanPhysiNode { SScanPhysiNode scan; uint8_t scanFlag; // denotes reversed scan of data or not @@ -209,10 +217,14 @@ typedef struct SExchangePhysiNode { SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; -typedef struct SIntervalPhysiNode { +typedef struct SWinodwPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; +} SWinodwPhysiNode; + +typedef struct SIntervalPhysiNode { + SWinodwPhysiNode window; int64_t interval; int64_t offset; int64_t sliding; @@ -221,6 +233,11 @@ typedef struct SIntervalPhysiNode { SFillNode* pFill; } SIntervalPhysiNode; +typedef struct SSessionWinodwPhysiNode { + SWinodwPhysiNode window; + int64_t gap; +} SSessionWinodwPhysiNode; + typedef struct SDataSinkNode { ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; @@ -243,6 +260,7 @@ typedef struct SSubplan { ESubplanType subplanType; int32_t msgType; // message type for subplan, used to denote the send message type to vnode. int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. + char dbFName[TSDB_DB_FNAME_LEN]; SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node SQueryNodeStat execNodeStat; // only for scan subplan SNodeList* pChildren; // the datasource subplan,from which to fetch the result diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index a55a0e218da1bc50c5013892925fe3ccdf6984c9..616e24d67d0b915c40532c4722f88283ae169990 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -23,7 +23,8 @@ extern "C" { #include "nodes.h" #include "tmsg.h" -#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) * sizeof(SSchema))) +#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) +#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema))) #define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo))) typedef struct SRawExprNode { @@ -129,6 +130,7 @@ typedef struct SRealTableNode { STableNode table; // QUERY_NODE_REAL_TABLE struct STableMeta* pMeta; SVgroupsInfo* pVgroupList; + char useDbName[TSDB_DB_NAME_LEN]; } SRealTableNode; typedef struct STempTableNode { @@ -189,8 +191,8 @@ typedef struct SStateWindowNode { typedef struct SSessionWindowNode { ENodeType type; // QUERY_NODE_SESSION_WINDOW - int64_t gap; // gap between two session window(in microseconds) SNode* pCol; + SNode* pGap; // gap between two session window(in microseconds) } SSessionWindowNode; typedef struct SIntervalWindowNode { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 23bcdabb1b3cf54705ebf7afbcfe86542da8302f..074753472172485802c465261e9708a7efeda285 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -26,7 +26,7 @@ typedef struct SParseContext { uint64_t requestId; int32_t acctId; const char *db; - bool streamQuery; + bool topicQuery; void *pTransporter; SEpSet mgmtEpSet; const char *pSql; // sql string @@ -52,13 +52,17 @@ typedef struct SQuery { SSchema* pResSchema; SCmdMsgInfo* pCmdMsg; int32_t msgType; - bool streamQuery; + SArray* pDbList; + SArray* pTableList; + bool showRewrite; } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); void qDestroyQuery(SQuery* pQueryNode); +int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); + #ifdef __cplusplus } #endif diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 7eb9d038a516a95dd3cec56d1422f4f5f544ebb7..8db78fccf580807270fcb5cddea39dc2405bfb69 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -25,8 +25,11 @@ extern "C" { typedef struct SPlanContext { uint64_t queryId; int32_t acctId; + SEpSet mgmtEpSet; SNode* pAstRoot; + bool topicQuery; bool streamQuery; + bool showRewrite; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b6f42eeee0ed23a98d595e03c786d9103feb6628..24953c668418b66af91814e4741005d0d87b4ec8 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -25,7 +25,7 @@ extern "C" { #include "tlog.h" #include "tmsg.h" -enum { +typedef enum { JOB_TASK_STATUS_NULL = 0, JOB_TASK_STATUS_NOT_START = 1, JOB_TASK_STATUS_EXECUTING, @@ -35,12 +35,12 @@ enum { JOB_TASK_STATUS_CANCELLING, JOB_TASK_STATUS_CANCELLED, JOB_TASK_STATUS_DROPPING, -}; +} EJobTaskType; -enum { +typedef enum { TASK_TYPE_PERSISTENT = 1, TASK_TYPE_TEMP, -}; +} ETaskType; typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema @@ -150,6 +150,8 @@ int32_t cleanupTaskQueue(); */ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); +int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *ctx); + /** * Asynchronously send message to server, after the response received, the callback will be incured. * @@ -169,6 +171,9 @@ const SSchema* tGetTbnameColumnSchema(); bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); +char *jobTaskStatusStr(int32_t status); + +SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); @@ -178,6 +183,15 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE +#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) +#define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) +#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) +#define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) + +#define NEED_SCHEDULER_RETRY_ERROR(_code) ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) + +#define REQUEST_MAX_TRY_TIMES 5 + #define qFatal(...) \ do { \ if (qDebugFlag & DEBUG_FATAL) { \ diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index dd3103edf1baf4f161f7dd7781934c818a919059..0846841cef1b509edf2ccc189bf9e81453169aa1 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "tmsgcb.h" #include "trpc.h" @@ -27,6 +28,7 @@ enum { NODE_TYPE_VNODE = 1, NODE_TYPE_QNODE, NODE_TYPE_SNODE, + NODE_TYPE_MNODE, }; @@ -48,11 +50,7 @@ typedef struct { uint64_t numOfErrors; } SQWorkerStat; -typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); -typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *); - -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, - putReqToQueryQFp fp1, sendReqToDnodeFp fp2); +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index e55c0b9e7667e934236c771d8980ace53aa57ca9..12c03afcb6fc70d2923fe067dad1d6ecf6ef87d4 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -42,6 +42,21 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut); +int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t sqrtFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +int32_t sinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t cosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t tanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t asinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t acosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 56da9ece6fe8b5131d27fd47c81cc21888f4bf23..16a6ae32cfd3fbda83a9d20a5fb2858e74c38bd7 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -52,10 +52,10 @@ typedef struct SQueryProfileSummary { } SQueryProfileSummary; typedef struct SQueryResult { - int32_t code; - uint64_t numOfRows; - int32_t msgSize; - char *msg; + int32_t code; + uint64_t numOfRows; + int32_t msgSize; + char *msg; } SQueryResult; typedef struct STaskInfo { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h new file mode 100644 index 0000000000000000000000000000000000000000..177fe39397b12efdf3ffd221bf69932ffb0d15ac --- /dev/null +++ b/include/libs/stream/tstream.h @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdatablock.h" +#include "tmsg.h" +#include "tmsgcb.h" +#include "trpc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _TSTREAM_H_ +#define _TSTREAM_H_ + +enum { + STREAM_TASK_STATUS__RUNNING = 1, + STREAM_TASK_STATUS__STOP, +}; + +#if 0 +// pipe -> fetch/pipe queue +// merge -> merge queue +// write -> write queue +enum { + TASK_DISPATCH_MSG__SND_PIPE = 1, + TASK_DISPATCH_MSG__SND_MERGE, + TASK_DISPATCH_MSG__VND_PIPE, + TASK_DISPATCH_MSG__VND_MERGE, + TASK_DISPATCH_MSG__VND_WRITE, +}; +#endif + +typedef struct { + int32_t nodeId; // 0 for snode + SEpSet epSet; +} SStreamTaskEp; + +typedef struct { + void* inputHandle; + void* executor; +} SStreamRunner; + +typedef struct { + int8_t parallelizable; + char* qmsg; + // followings are not applicable to encoder and decoder + int8_t numOfRunners; + SStreamRunner* runners; +} STaskExec; + +typedef struct { + int8_t reserved; +} STaskDispatcherInplace; + +typedef struct { + int32_t nodeId; + SEpSet epSet; +} STaskDispatcherFixedEp; + +typedef struct { + int8_t hashMethod; + SArray* info; +} STaskDispatcherShuffle; + +typedef struct { + int8_t reserved; + // not applicable to encoder and decoder + SHashObj* pHash; // groupId to tbuid +} STaskSinkTb; + +typedef struct { + int8_t reserved; +} STaskSinkSma; + +typedef struct { + int8_t reserved; +} STaskSinkFetch; + +typedef struct { + int8_t reserved; +} STaskSinkShow; + +enum { + TASK_SOURCE__SCAN = 1, + TASK_SOURCE__PIPE, + TASK_SOURCE__MERGE, +}; + +enum { + TASK_EXEC__NONE = 1, + TASK_EXEC__PIPE, + TASK_EXEC__MERGE, +}; + +enum { + TASK_DISPATCH__NONE = 1, + TASK_DISPATCH__INPLACE, + TASK_DISPATCH__FIXED, + TASK_DISPATCH__SHUFFLE, +}; + +enum { + TASK_SINK__NONE = 1, + TASK_SINK__TABLE, + TASK_SINK__SMA, + TASK_SINK__FETCH, + TASK_SINK__SHOW, +}; + +typedef struct { + int64_t streamId; + int32_t taskId; + int8_t status; + + int8_t sourceType; + int8_t execType; + int8_t sinkType; + int8_t dispatchType; + int16_t dispatchMsgType; + int32_t downstreamTaskId; + + int32_t nodeId; + SEpSet epSet; + + // source preprocess + + // exec + STaskExec exec; + + // local sink + union { + STaskSinkTb tbSink; + STaskSinkSma smaSink; + STaskSinkFetch fetchSink; + STaskSinkShow showSink; + }; + + // dispatch + union { + STaskDispatcherInplace inplaceDispatcher; + STaskDispatcherFixedEp fixedEpDispatcher; + STaskDispatcherShuffle shuffleDispatcher; + }; + + // state storage + +} SStreamTask; + +SStreamTask* tNewSStreamTask(int64_t streamId); +int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask); +int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask); +void tFreeSStreamTask(SStreamTask* pTask); + +typedef struct { + // SMsgHead head; + SStreamTask* task; +} SStreamTaskDeployReq; + +typedef struct { + int32_t reserved; +} SStreamTaskDeployRsp; + +typedef struct { + // SMsgHead head; + int64_t streamId; + int32_t taskId; + SArray* data; // SArray +} SStreamTaskExecReq; + +int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq); +void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq); +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq); + +typedef struct { + int32_t reserved; +} SStreamTaskExecRsp; + +typedef struct { + // SMsgHead head; + int64_t streamId; + int64_t version; + SArray* res; // SArray +} SStreamSinkReq; + +int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId); + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef _TSTREAM_H_ */ diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 68ca9eff17acc03c0500997f51b3f7a3172a91c8..a2f88490f05f8b579b40f8209d2dab6b7ee934ce 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -35,6 +35,7 @@ typedef enum { TAOS_SYNC_STATE_FOLLOWER = 100, TAOS_SYNC_STATE_CANDIDATE = 101, TAOS_SYNC_STATE_LEADER = 102, + TAOS_SYNC_STATE_ERROR = 103, } ESyncState; typedef struct SSyncBuffer { @@ -68,17 +69,20 @@ typedef struct SSnapshot { typedef struct SSyncFSM { void* data; - // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result + // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); - // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result + // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); // when log entry is updated by a new one, FpRollBackCb is called // user can do something to roll back. for example, delete data from tsdb, or just ignore it - void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); // user should implement this function, use "data" to take snapshot into "snapshot" int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); @@ -157,9 +161,14 @@ void syncCleanUp(); int64_t syncStart(const SSyncInfo* pSyncInfo); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); +int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); ESyncState syncGetMyRole(int64_t rid); -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); + +// propose with sequence number, to implement linearizable semantics +int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum); + +// for compatibility, the same as syncPropose +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); extern int32_t sDebugFlag; diff --git a/source/libs/transport/inc/rpcHead.h b/include/libs/transport/rpcHead.h similarity index 100% rename from source/libs/transport/inc/rpcHead.h rename to include/libs/transport/rpcHead.h diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 6bc28e3ea03719e16f75a0aab98fa9b9228eed1f..8125de76474ee6e8f08ee4e2135b436807a835f1 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -38,16 +38,24 @@ typedef struct SRpcConnInfo { typedef struct SRpcMsg { tmsg_t msgType; - tmsg_t expectMsgType; void * pCont; int contLen; int32_t code; - void * handle; // rpc handle returned to app - void * ahandle; // app handle set by client - int noResp; // has response or not(default 0 indicate resp); + void * handle; // rpc handle returned to app + void * ahandle; // app handle set by client + int noResp; // has response or not(default 0, 0: resp, 1: no resp); + int persistHandle; // persist handle or not } SRpcMsg; +typedef struct { + char user[TSDB_USER_LEN]; + SRpcMsg rpcMsg; + int32_t rspLen; + void * pRsp; + void * pNode; +} SNodeMsg; + typedef struct SRpcInit { uint16_t localPort; // local port char * label; // for debug purpose @@ -69,17 +77,26 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); - // call back to keep conn or not - bool (*pfp)(void *parent, tmsg_t msgType); + void *parent; +} SRpcInit; - // to support Send messages multiple times on a link - void *(*mfp)(void *parent, tmsg_t msgType); +typedef struct { + void *val; + int32_t (*clone)(void *src, void **dst); + void (*freeFunc)(const void *arg); +} SRpcCtxVal; - // call back to handle except when query/fetch in progress - bool (*efp)(void *parent, tmsg_t msgType); +typedef struct { + int32_t msgType; + void *val; + int32_t (*clone)(void *src, void **dst); + void (*freeFunc)(const void *arg); +} SRpcBrokenlinkVal; - void *parent; -} SRpcInit; +typedef struct { + SHashObj * args; + SRpcBrokenlinkVal brokenVal; +} SRpcCtx; int32_t rpcInit(); void rpcCleanup(); @@ -89,16 +106,17 @@ void * rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void * rpcReallocCont(void *ptr, int contLen); void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); -void rpcSendResponse(const SRpcMsg *pMsg); -void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); -int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -int rpcReportProgress(void *pConn, char *pCont, int contLen); -void rpcCancelRequest(int64_t rid); - +void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); + +void rpcSendResponse(const SRpcMsg *pMsg); +void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); +int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); +void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int rpcReportProgress(void *pConn, char *pCont, int contLen); +void rpcCancelRequest(int64_t rid); +void rpcRegisterBrokenLinkArg(SRpcMsg *msg); // just release client conn to rpc instance, no close sock -void rpcReleaseHandle(void *handle, int8_t type); - +void rpcReleaseHandle(void *handle, int8_t type); // void rpcRefHandle(void *handle, int8_t type); void rpcUnrefHandle(void *handle, int8_t type); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 32fcd95948d7ef1e481f04d484c9e54070c47f4e..8b1ac3ed8d3513f03bffcf71ed4af17bfe650adc 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -61,30 +61,45 @@ extern "C" { } \ } -#define WAL_HEAD_VER 0 +#define WAL_HEAD_VER 0 #define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) -#define WAL_LOG_SUFFIX "log" +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) +#define WAL_LOG_SUFFIX "log" #define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_MAGIC 0xFAFBFCFDULL +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) +#define WAL_MAGIC 0xFAFBFCFDULL #define WAL_CUR_FAILED 1 #pragma pack(push, 1) -typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType; +typedef enum { + TAOS_WAL_NOLOG = 0, + TAOS_WAL_WRITE = 1, + TAOS_WAL_FSYNC = 2, +} EWalType; + +// used by sync module +typedef struct { + int8_t isWeek; + uint64_t seqNum; + uint64_t term; +} SSyncLogMeta; typedef struct SWalReadHead { int8_t headVer; - int16_t msgType; int8_t reserved; + int16_t msgType; int32_t len; int64_t ingestTs; // not implemented int64_t version; - char body[]; + + // sync meta + SSyncLogMeta syncMeta; + + char body[]; } SWalReadHead; typedef struct { @@ -117,17 +132,17 @@ typedef struct SWal { SWalCfg cfg; int32_t fsyncSeq; // meta - SWalVer vers; + SWalVer vers; TdFilePtr pWriteLogTFile; TdFilePtr pWriteIdxTFile; - int32_t writeCur; - SArray *fileInfoSet; + int32_t writeCur; + SArray *fileInfoSet; // status int64_t totSize; int64_t lastRollSeq; // ctl - int64_t refId; - pthread_mutex_t mutex; + int64_t refId; + TdThreadMutex mutex; // path char path[WAL_PATH_LEN]; // reusable write head @@ -158,6 +173,8 @@ int32_t walAlter(SWal *, SWalCfg *pCfg); void walClose(SWal *); // write +int64_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body, + int32_t bodyLen); int64_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen); void walFsync(SWal *, bool force); diff --git a/include/os/os.h b/include/os/os.h index 9118a92e11beff1c0403616632be4ff4c5493195..b05bfab6d02b5bdd8e1c7fe8be8ad39f10feed5b 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -22,7 +22,6 @@ extern "C" { #include #include -#include #include #include @@ -83,6 +82,7 @@ extern "C" { #include "osMath.h" #include "osMemory.h" #include "osRand.h" +#include "osThread.h" #include "osSemaphore.h" #include "osSignal.h" #include "osSleep.h" @@ -90,7 +90,6 @@ extern "C" { #include "osString.h" #include "osSysinfo.h" #include "osSystem.h" -#include "osThread.h" #include "osTime.h" #include "osTimer.h" #include "osTimezone.h" diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index bd3d372fc0cfb0138fc63b870c427179c4cac7db..5d45ced3edfafce8797e8386588ca01935a8d66c 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -20,337 +20,96 @@ extern "C" { #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #define atomic_load_8(ptr) (*(char volatile*)(ptr)) - #define atomic_load_16(ptr) (*(short volatile*)(ptr)) - #define atomic_load_32(ptr) (*(long volatile*)(ptr)) - #define atomic_load_64(ptr) (*(__int64 volatile*)(ptr)) - #define atomic_load_ptr(ptr) (*(void* volatile*)(ptr)) - - #define atomic_store_8(ptr, val) ((*(char volatile*)(ptr)) = (char)(val)) - #define atomic_store_16(ptr, val) ((*(short volatile*)(ptr)) = (short)(val)) - #define atomic_store_32(ptr, val) ((*(long volatile*)(ptr)) = (long)(val)) - #define atomic_store_64(ptr, val) ((*(__int64 volatile*)(ptr)) = (__int64)(val)) - #define atomic_store_ptr(ptr, val) ((*(void* volatile*)(ptr)) = (void*)(val)) - - #define atomic_exchange_8(ptr, val) _InterlockedExchange8((char volatile*)(ptr), (char)(val)) - #define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val)) - #define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val)) - #define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _WIN64 - #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) - #else - #define atomic_exchange_ptr(ptr, val) _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) - #endif - - #ifdef _TD_GO_DLL_ - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap - #else - #define atomic_val_compare_exchange_8(ptr, oldval, newval) _InterlockedCompareExchange8((char volatile*)(ptr), (char)(newval), (char)(oldval)) - #endif - #define atomic_val_compare_exchange_16(ptr, oldval, newval) _InterlockedCompareExchange16((short volatile*)(ptr), (short)(newval), (short)(oldval)) - #define atomic_val_compare_exchange_32(ptr, oldval, newval) _InterlockedCompareExchange((long volatile*)(ptr), (long)(newval), (long)(oldval)) - #define atomic_val_compare_exchange_64(ptr, oldval, newval) _InterlockedCompareExchange64((__int64 volatile*)(ptr), (__int64)(newval), (__int64)(oldval)) - #define atomic_val_compare_exchange_ptr(ptr, oldval, newval) _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)) - - char interlocked_add_fetch_8(char volatile *ptr, char val); - short interlocked_add_fetch_16(short volatile *ptr, short val); - long interlocked_add_fetch_32(long volatile *ptr, long val); - __int64 interlocked_add_fetch_64(__int64 volatile *ptr, __int64 val); - - char interlocked_and_fetch_8(char volatile* ptr, char val); - short interlocked_and_fetch_16(short volatile* ptr, short val); - long interlocked_and_fetch_32(long volatile* ptr, long val); - __int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val); - - __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val); - - char interlocked_or_fetch_8(char volatile* ptr, char val); - short interlocked_or_fetch_16(short volatile* ptr, short val); - long interlocked_or_fetch_32(long volatile* ptr, long val); - __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val); - - char interlocked_xor_fetch_8(char volatile* ptr, char val); - short interlocked_xor_fetch_16(short volatile* ptr, short val); - long interlocked_xor_fetch_32(long volatile* ptr, long val); - __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val); - - __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val); - - #define atomic_add_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_add_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), (short)(val)) - #define atomic_add_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), (long)(val)) - #define atomic_add_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _TD_GO_DLL_ - #define atomic_fetch_add_8 __sync_fetch_and_ad - #define atomic_fetch_add_16 __sync_fetch_and_add - #else - #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) - #endif - #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) - #define atomic_fetch_add_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), (long)(val)) - #define atomic_fetch_add_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_sub_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), -(char)(val)) - #define atomic_sub_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), -(short)(val)) - #define atomic_sub_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), -(long)(val)) - #define atomic_sub_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), -(__int64)(val)) - - #define atomic_fetch_sub_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), -(char)(val)) - #define atomic_fetch_sub_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), -(short)(val)) - #define atomic_fetch_sub_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), -(long)(val)) - #define atomic_fetch_sub_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), -(__int64)(val)) - - #define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val)) - #define atomic_and_fetch_32(ptr, val) interlocked_and_fetch_32((long volatile*)(ptr), (long)(val)) - #define atomic_and_fetch_64(ptr, val) interlocked_and_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val)) - #define atomic_fetch_and_32(ptr, val) _InterlockedAnd((long volatile*)(ptr), (long)(val)) - #define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val)) - #define atomic_or_fetch_32(ptr, val) interlocked_or_fetch_32((long volatile*)(ptr), (long)(val)) - #define atomic_or_fetch_64(ptr, val) interlocked_or_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val)) - #define atomic_fetch_or_32(ptr, val) _InterlockedOr((long volatile*)(ptr), (long)(val)) - #define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val)) - #define atomic_xor_fetch_32(ptr, val) interlocked_xor_fetch_32((long volatile*)(ptr), (long)(val)) - #define atomic_xor_fetch_64(ptr, val) interlocked_xor_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val)) - #define atomic_fetch_xor_32(ptr, val) _InterlockedXor((long volatile*)(ptr), (long)(val)) - #define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val)) - - #ifdef _WIN64 - #define atomic_add_fetch_ptr atomic_add_fetch_64 - #define atomic_fetch_add_ptr atomic_fetch_add_64 - #define atomic_sub_fetch_ptr atomic_sub_fetch_64 - #define atomic_fetch_sub_ptr atomic_fetch_sub_64 - #define atomic_and_fetch_ptr atomic_and_fetch_64 - #define atomic_fetch_and_ptr atomic_fetch_and_64 - #define atomic_or_fetch_ptr atomic_or_fetch_64 - #define atomic_fetch_or_ptr atomic_fetch_or_64 - #define atomic_xor_fetch_ptr atomic_xor_fetch_64 - #define atomic_fetch_xor_ptr atomic_fetch_xor_64 - #else - #define atomic_add_fetch_ptr atomic_add_fetch_32 - #define atomic_fetch_add_ptr atomic_fetch_add_32 - #define atomic_sub_fetch_ptr atomic_sub_fetch_32 - #define atomic_fetch_sub_ptr atomic_fetch_sub_32 - #define atomic_and_fetch_ptr atomic_and_fetch_32 - #define atomic_fetch_and_ptr atomic_fetch_and_32 - #define atomic_or_fetch_ptr atomic_or_fetch_32 - #define atomic_fetch_or_ptr atomic_fetch_or_32 - #define atomic_xor_fetch_ptr atomic_xor_fetch_32 - #define atomic_fetch_xor_ptr atomic_fetch_xor_32 - #endif -#elif defined(_TD_NINGSI_60) - /* - * type __sync_fetch_and_add (type *ptr, type value); - * type __sync_fetch_and_sub (type *ptr, type value); - * type __sync_fetch_and_or (type *ptr, type value); - * type __sync_fetch_and_and (type *ptr, type value); - * type __sync_fetch_and_xor (type *ptr, type value); - * type __sync_fetch_and_nand (type *ptr, type value); - * type __sync_add_and_fetch (type *ptr, type value); - * type __sync_sub_and_fetch (type *ptr, type value); - * type __sync_or_and_fetch (type *ptr, type value); - * type __sync_and_and_fetch (type *ptr, type value); - * type __sync_xor_and_fetch (type *ptr, type value); - * type __sync_nand_and_fetch (type *ptr, type value); - * - * bool __sync_bool_compare_and_swap (type*ptr, type oldval, type newval, ...) - * type __sync_val_compare_and_swap (type *ptr, type oldval, ?type newval, ...) - * */ - - #define atomic_load_8(ptr) __sync_fetch_and_add((ptr), 0) - #define atomic_load_16(ptr) __sync_fetch_and_add((ptr), 0) - #define atomic_load_32(ptr) __sync_fetch_and_add((ptr), 0) - #define atomic_load_64(ptr) __sync_fetch_and_add((ptr), 0) - #define atomic_load_ptr(ptr) __sync_fetch_and_add((ptr), 0) - - #define atomic_store_8(ptr, val) (*(ptr)=(val)) - #define atomic_store_16(ptr, val) (*(ptr)=(val)) - #define atomic_store_32(ptr, val) (*(ptr)=(val)) - #define atomic_store_64(ptr, val) (*(ptr)=(val)) - #define atomic_store_ptr(ptr, val) (*(ptr)=(val)) - - int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ); - int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ); - int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ); - int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ); - void* atomic_exchange_ptr_impl( void **ptr, void *val ); - - #define atomic_exchange_8(ptr, val) atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val) - #define atomic_exchange_16(ptr, val) atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val) - #define atomic_exchange_32(ptr, val) atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val) - #define atomic_exchange_64(ptr, val) atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val) - #define atomic_exchange_ptr(ptr, val) atomic_exchange_ptr_impl((void **)ptr, (void*)val) - - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap - - #define atomic_add_fetch_8(ptr, val) __sync_add_and_fetch((ptr), (val)) - #define atomic_add_fetch_16(ptr, val) __sync_add_and_fetch((ptr), (val)) - #define atomic_add_fetch_32(ptr, val) __sync_add_and_fetch((ptr), (val)) - #define atomic_add_fetch_64(ptr, val) __sync_add_and_fetch((ptr), (val)) - #define atomic_add_fetch_ptr(ptr, val) __sync_add_and_fetch((ptr), (val)) - - #define atomic_fetch_add_8(ptr, val) __sync_fetch_and_add((ptr), (val)) - #define atomic_fetch_add_16(ptr, val) __sync_fetch_and_add((ptr), (val)) - #define atomic_fetch_add_32(ptr, val) __sync_fetch_and_add((ptr), (val)) - #define atomic_fetch_add_64(ptr, val) __sync_fetch_and_add((ptr), (val)) - #define atomic_fetch_add_ptr(ptr, val) __sync_fetch_and_add((ptr), (val)) - - #define atomic_sub_fetch_8(ptr, val) __sync_sub_and_fetch((ptr), (val)) - #define atomic_sub_fetch_16(ptr, val) __sync_sub_and_fetch((ptr), (val)) - #define atomic_sub_fetch_32(ptr, val) __sync_sub_and_fetch((ptr), (val)) - #define atomic_sub_fetch_64(ptr, val) __sync_sub_and_fetch((ptr), (val)) - #define atomic_sub_fetch_ptr(ptr, val) __sync_sub_and_fetch((ptr), (val)) - - #define atomic_fetch_sub_8(ptr, val) __sync_fetch_and_sub((ptr), (val)) - #define atomic_fetch_sub_16(ptr, val) __sync_fetch_and_sub((ptr), (val)) - #define atomic_fetch_sub_32(ptr, val) __sync_fetch_and_sub((ptr), (val)) - #define atomic_fetch_sub_64(ptr, val) __sync_fetch_and_sub((ptr), (val)) - #define atomic_fetch_sub_ptr(ptr, val) __sync_fetch_and_sub((ptr), (val)) - - #define atomic_and_fetch_8(ptr, val) __sync_and_and_fetch((ptr), (val)) - #define atomic_and_fetch_16(ptr, val) __sync_and_and_fetch((ptr), (val)) - #define atomic_and_fetch_32(ptr, val) __sync_and_and_fetch((ptr), (val)) - #define atomic_and_fetch_64(ptr, val) __sync_and_and_fetch((ptr), (val)) - #define atomic_and_fetch_ptr(ptr, val) __sync_and_and_fetch((ptr), (val)) - - #define atomic_fetch_and_8(ptr, val) __sync_fetch_and_and((ptr), (val)) - #define atomic_fetch_and_16(ptr, val) __sync_fetch_and_and((ptr), (val)) - #define atomic_fetch_and_32(ptr, val) __sync_fetch_and_and((ptr), (val)) - #define atomic_fetch_and_64(ptr, val) __sync_fetch_and_and((ptr), (val)) - #define atomic_fetch_and_ptr(ptr, val) __sync_fetch_and_and((ptr), (val)) - - #define atomic_or_fetch_8(ptr, val) __sync_or_and_fetch((ptr), (val)) - #define atomic_or_fetch_16(ptr, val) __sync_or_and_fetch((ptr), (val)) - #define atomic_or_fetch_32(ptr, val) __sync_or_and_fetch((ptr), (val)) - #define atomic_or_fetch_64(ptr, val) __sync_or_and_fetch((ptr), (val)) - #define atomic_or_fetch_ptr(ptr, val) __sync_or_and_fetch((ptr), (val)) - - #define atomic_fetch_or_8(ptr, val) __sync_fetch_and_or((ptr), (val)) - #define atomic_fetch_or_16(ptr, val) __sync_fetch_and_or((ptr), (val)) - #define atomic_fetch_or_32(ptr, val) __sync_fetch_and_or((ptr), (val)) - #define atomic_fetch_or_64(ptr, val) __sync_fetch_and_or((ptr), (val)) - #define atomic_fetch_or_ptr(ptr, val) __sync_fetch_and_or((ptr), (val)) - - #define atomic_xor_fetch_8(ptr, val) __sync_xor_and_fetch((ptr), (val)) - #define atomic_xor_fetch_16(ptr, val) __sync_xor_and_fetch((ptr), (val)) - #define atomic_xor_fetch_32(ptr, val) __sync_xor_and_fetch((ptr), (val)) - #define atomic_xor_fetch_64(ptr, val) __sync_xor_and_fetch((ptr), (val)) - #define atomic_xor_fetch_ptr(ptr, val) __sync_xor_and_fetch((ptr), (val)) - - #define atomic_fetch_xor_8(ptr, val) __sync_fetch_and_xor((ptr), (val)) - #define atomic_fetch_xor_16(ptr, val) __sync_fetch_and_xor((ptr), (val)) - #define atomic_fetch_xor_32(ptr, val) __sync_fetch_and_xor((ptr), (val)) - #define atomic_fetch_xor_64(ptr, val) __sync_fetch_and_xor((ptr), (val)) - #define atomic_fetch_xor_ptr(ptr, val) __sync_fetch_and_xor((ptr), (val)) - -#else - #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - - #define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap - - #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +// If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. +#ifndef ALLOW_FORBID_FUNC + #define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID + #define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID + #define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID + #define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID + #define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID + #define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID + #define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID + #define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID + #define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID #endif +int8_t atomic_load_8(int8_t volatile *ptr); +int16_t atomic_load_16(int16_t volatile *ptr); +int32_t atomic_load_32(int32_t volatile *ptr); +int64_t atomic_load_64(int64_t volatile *ptr); +void* atomic_load_ptr(void *ptr); +void atomic_store_8(int8_t volatile *ptr, int8_t val); +void atomic_store_16(int16_t volatile *ptr, int16_t val); +void atomic_store_32(int32_t volatile *ptr, int32_t val); +void atomic_store_64(int64_t volatile *ptr, int64_t val); +void atomic_store_ptr(void *ptr, void *val); +int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); +void* atomic_exchange_ptr(void *ptr, void *val); +int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); +int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); +int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval); +int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval); +void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval); +int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_add_fetch_ptr(void *ptr, int32_t val); +int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_add_ptr(void *ptr, int32_t val); +int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_sub_fetch_ptr(void *ptr, int32_t val); +int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_sub_ptr(void *ptr, int32_t val); +int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_and_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_and_ptr(void *ptr, void *val); +int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_or_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_or_ptr(void *ptr, void *val); +int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_xor_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_xor_ptr(void *ptr, void *val); + #ifdef __cplusplus } #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 6cf28fb8789013c64440917b25ed86825782f753..d3597cab36aa27e19360f290ec841c54a3e96ccd 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -17,6 +17,7 @@ #define _TD_OS_DIR_H_ // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define opendir OPENDIR_FUNC_TAOS_FORBID #define readdir READDIR_FUNC_TAOS_FORBID diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 1426ba87f652552c9cf85cc98777654494648cb1..ebf4c360dd7c5c65672d70248b060960517a5ff4 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -45,6 +45,7 @@ extern SDiskSpace tsTempSpace; void osInit(); void osUpdate(); +void osCleanup(); bool osLogSpaceAvailable(); void osSetTimezone(const char *timezone); diff --git a/include/os/osFile.h b/include/os/osFile.h index 508a52267901994da5bb1d8468b7bf48992da357..143b4bf2f8509fce4731c89e9e75b3212bac678e 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -23,6 +23,7 @@ extern "C" { #include "osSocket.h" // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following sectio #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID diff --git a/include/os/osLocale.h b/include/os/osLocale.h index ddafd2e93c5f549beb30304d17d96dfe408ae518..74922cc0b9fcfe3312458583a4c38f23391d1305 100644 --- a/include/os/osLocale.h +++ b/include/os/osLocale.h @@ -23,6 +23,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define setlocale SETLOCALE_FUNC_TAOS_FORBID #endif diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 6100419035ff9901bf6a6d5c3871496d0edcd533..62ac82782c5d6c77d27bea9df825fc8e5be69e60 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -20,12 +20,25 @@ extern "C" { #endif -#define tfree(x) \ - do { \ - if (x) { \ - free((void *)(x)); \ - (x) = 0; \ - } \ +// If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following sectio +#ifndef ALLOW_FORBID_FUNC + #define malloc MALLOC_FUNC_TAOS_FORBID + #define calloc CALLOC_FUNC_TAOS_FORBID + #define realloc REALLOC_FUNC_TAOS_FORBID + #define free FREE_FUNC_TAOS_FORBID +#endif + +void *taosMemoryMalloc(int32_t size); +void *taosMemoryCalloc(int32_t num, int32_t size); +void *taosMemoryRealloc(void *ptr, int32_t size); +void taosMemoryFree(const void *ptr); +int32_t taosMemorySize(void *ptr); + +#define taosMemoryFreeClear(ptr) \ + do { \ + taosMemoryFree(ptr); \ + (ptr)=NULL; \ } while (0) #ifdef __cplusplus diff --git a/include/os/osRand.h b/include/os/osRand.h index 09e1f1b41d36dceac1530c961775c77ba6d20fdb..bce2c08a2ec16468b9c8a8ae7a4764e8838fb492 100644 --- a/include/os/osRand.h +++ b/include/os/osRand.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define rand RAND_FUNC_TAOS_FORBID #define srand SRAND_FUNC_TAOS_FORBID diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 4bac81754db48f65295e0489dd93bd9982095a50..594daf1bf38b2fc0ca6ed522006eeca46012dbb1 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -20,7 +20,6 @@ extern "C" { #endif -#include #include #if defined (_TD_DARWIN_64) @@ -38,25 +37,25 @@ extern "C" { #endif #if defined (_TD_DARWIN_64) -// #define pthread_rwlock_t pthread_mutex_t -// #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL) -// #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock) -// #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock) -// #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock) -// #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock) +// #define TdThreadRwlock TdThreadMutex +// #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL) +// #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock) +// #define taosThreadRwlockWrlock(lock) taosThreadMutexLock(lock) +// #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock) +// #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock) - #define pthread_spinlock_t pthread_mutex_t - #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL) - #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock) - #define pthread_spin_lock(lock) pthread_mutex_lock(lock) - #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock) + #define TdThreadSpinlock TdThreadMutex + #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL) + #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock) + #define taosThreadSpinLock(lock) taosThreadMutexLock(lock) + #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock) #endif -bool taosCheckPthreadValid(pthread_t thread); +bool taosCheckPthreadValid(TdThread thread); int64_t taosGetSelfPthreadId(); -int64_t taosGetPthreadId(pthread_t thread); -void taosResetPthread(pthread_t* thread); -bool taosComparePthread(pthread_t first, pthread_t second); +int64_t taosGetPthreadId(TdThread thread); +void taosResetPthread(TdThread* thread); +bool taosComparePthread(TdThread first, TdThread second); int32_t taosGetPId(); int32_t taosGetAppName(char* name, int32_t* len); diff --git a/include/os/osSleep.h b/include/os/osSleep.h index feb94729954e73cc74e1a6b33ec8621a9be10a44..1d0ccf32bd272ab60c42efae47bf8eba2050fa4f 100644 --- a/include/os/osSleep.h +++ b/include/os/osSleep.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define Sleep SLEEP_FUNC_TAOS_FORBID #define sleep SLEEP_FUNC_TAOS_FORBID diff --git a/include/os/osSocket.h b/include/os/osSocket.h index fb330a9e2fc62cc90f9047ace4488a954a51f50b..7af8dd37bffb9bad5e91abe21936e272442c2889 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -17,6 +17,7 @@ #define _TD_OS_SOCKET_H_ // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define socket SOCKET_FUNC_TAOS_FORBID #define bind BIND_FUNC_TAOS_FORBID @@ -50,11 +51,29 @@ extern "C" { #endif -#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) +#if defined(WINDOWS) #define htobe64 htonll - #if defined(_TD_GO_DLL_) - uint64_t htonll(uint64_t val); +#endif + +#if defined(WINDOWS) + #define TAOS_EPOLL_WAIT_TIME 100 + typedef SOCKET eventfd_t; + #define eventfd(a, b) -1 + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + #ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) #endif +#elif defined(_TD_DARWIN_64) + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) +#else + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) taosCloseSocket(pollFd) #endif #if defined(_TD_DARWIN_64) @@ -66,12 +85,12 @@ extern "C" { # define htole16(x) OSSwapHostToLittleInt16(x) # define be16toh(x) OSSwapBigToHostInt16(x) # define le16toh(x) OSSwapLittleToHostInt16(x) - + # define htobe32(x) OSSwapHostToBigInt32(x) # define htole32(x) OSSwapHostToLittleInt32(x) # define be32toh(x) OSSwapBigToHostInt32(x) # define le32toh(x) OSSwapLittleToHostInt32(x) - + # define htobe64(x) OSSwapHostToBigInt64(x) # define htole64(x) OSSwapHostToLittleInt64(x) # define be64toh(x) OSSwapBigToHostInt64(x) @@ -85,6 +104,17 @@ extern "C" { #define TAOS_EPOLL_WAIT_TIME 500 +typedef int32_t SocketFd; +typedef SocketFd EpollFd; + +typedef struct TdSocket { +#if SOCKET_WITH_LOCK + TdThreadRwlock rwlock; +#endif + int refId; + SocketFd fd; +} *TdSocketPtr, TdSocket; + typedef struct TdSocketServer *TdSocketServerPtr; typedef struct TdSocket *TdSocketPtr; typedef struct TdEpoll *TdEpollPtr; @@ -93,6 +123,7 @@ int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosCloseSocketNoCheck1(SocketFd fd); int32_t taosCloseSocket(TdSocketPtr *ppSocket); int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); int32_t taosShutDownSocketRD(TdSocketPtr pSocket); diff --git a/include/os/osString.h b/include/os/osString.h index 9c6d523ab2f239fd8c66cca9a68de5fff76c317f..66d69a849c94d6a2c6c136d771c6ef0db3e15eef 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -24,6 +24,7 @@ typedef wchar_t TdWchar; typedef int32_t TdUcs4; // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID #define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID @@ -35,7 +36,7 @@ typedef int32_t TdUcs4; #define wctomb WCTOMB_FUNC_TAOS_FORBID #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID - #define wchar_t WCHAR_T_FUNC_TAOS_FORBID + #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #endif #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/include/os/osSystem.h b/include/os/osSystem.h index f130e9d8f19c76171c44361fc8676b10bc650080..15959a2d8c690813d777577075a373dfc501637f 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define popen POPEN_FUNC_TAOS_FORBID #define pclose PCLOSE_FUNC_TAOS_FORBID @@ -28,7 +29,6 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif -int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize); void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); diff --git a/include/os/osThread.h b/include/os/osThread.h index cccc13755d3a33f0ecaa1173df391228d3c90da9..6d8ff1de3eb39d7bb1c3c7e8be84415db638dd73 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -22,6 +22,95 @@ extern "C" { #endif +typedef pthread_t TdThread; +typedef pthread_spinlock_t TdThreadSpinlock; +typedef pthread_mutex_t TdThreadMutex; +typedef pthread_mutexattr_t TdThreadMutexAttr; +typedef pthread_rwlock_t TdThreadRwlock; +typedef pthread_attr_t TdThreadAttr; +typedef pthread_once_t TdThreadOnce; +typedef pthread_rwlockattr_t TdThreadRwlockAttr; +typedef pthread_cond_t TdThreadCond; +typedef pthread_condattr_t TdThreadCondAttr; + +#define taosThreadCleanupPush pthread_cleanup_push +#define taosThreadCleanupPop pthread_cleanup_pop + +// If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. +#ifndef ALLOW_FORBID_FUNC + #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID + #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID + #define pthread_mutex_t PTHREAD_MUTEX_T_TYPE_TAOS_FORBID + #define pthread_mutexattr_t PTHREAD_MUTEXATTR_T_TYPE_TAOS_FORBID + #define pthread_rwlock_t PTHREAD_RWLOCK_T_TYPE_TAOS_FORBID + #define pthread_attr_t PTHREAD_ATTR_T_TYPE_TAOS_FORBID + #define pthread_once_t PTHREAD_ONCE_T_TYPE_TAOS_FORBID + #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID + #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID + #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID + #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID + #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID + #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID + #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID + #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID + #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID + #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID + #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID + #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID + #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID + #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID + #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID + #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID + #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID + #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID + #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID + #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID + #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID + #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID + #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID + #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID + #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID + #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID + #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID + #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID + #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID + #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID +#endif + +int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared); +int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr); +int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock); +int32_t taosThreadMutexDestroy(TdThreadMutex * mutex); +int32_t taosThreadSpinLock(TdThreadSpinlock *lock); +int32_t taosThreadMutexLock(TdThreadMutex *mutex); +int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock); +int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock); +int32_t taosThreadMutexUnlock(TdThreadMutex *mutex); +int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock); +int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock); +void taosThreadTestCancel(void); +int32_t taosThreadAttrInit(TdThreadAttr *attr); +int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg); +int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void)); +int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState); +int32_t taosThreadAttrDestroy(TdThreadAttr *attr); +int32_t taosThreadJoin(TdThread thread, void **pValue); +int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr); +int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock); +int32_t taosThreadCondSignal(TdThreadCond *cond); +int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr); +int32_t taosThreadCondBroadcast(TdThreadCond *cond); +int32_t taosThreadCondDestroy(TdThreadCond *cond); +int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex); +TdThread taosThreadSelf(void); +int32_t taosThreadEqual(TdThread t1, TdThread t2); +int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset); +int32_t taosThreadCancel(TdThread thread); +int32_t taosThreadKill(TdThread thread, int sig); #ifdef __cplusplus } #endif diff --git a/include/os/osTime.h b/include/os/osTime.h index 4904eb9b514ca349cffc1be2e1574d706a3ccf4d..031b9d28f96405cc0fe71dd8f4cdab024df75539 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define strptime STRPTIME_FUNC_TAOS_FORBID #define gettimeofday GETTIMEOFDAY_FUNC_TAOS_FORBID @@ -33,11 +34,7 @@ extern "C" { #define CLOCK_REALTIME 0 - #ifdef _TD_GO_DLL_ - #define MILLISECOND_PER_SECOND (1000LL) - #else - #define MILLISECOND_PER_SECOND (1000i64) - #endif + #define MILLISECOND_PER_SECOND (1000i64) #else #define MILLISECOND_PER_SECOND ((int64_t)1000L) #endif diff --git a/include/os/osTimer.h b/include/os/osTimer.h index 9040113b232e6b10cf73b960e636112f8e2edf01..5fece432177e9a2f9923fb2507bdf339188eb1b0 100644 --- a/include/os/osTimer.h +++ b/include/os/osTimer.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define timer_create TIMER_CREATE_FUNC_TAOS_FORBID #define timer_settime TIMER_SETTIME_FUNC_TAOS_FORBID diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index ff015ef0b1221eeba361818f5e010bd7f43fb0fd..c8df8c3f3d50ec1a105d23214bb9579731e0db9f 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -20,6 +20,12 @@ extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. +#ifndef ALLOW_FORBID_FUNC + #define tzset TZSET_FUNC_TAOS_FORBID +#endif + void taosGetSystemTimezone(char *outTimezone); void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 7023691fe06959b51ea534518ab6c98e2334098b..87781b6313d73f3060afe90d6bba17197e8373bb 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -75,6 +75,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) #define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) +#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x010E) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) @@ -273,38 +274,23 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) +// mnode-sma +#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0400) +#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0401) +#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0402) + // dnode -#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) -#define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) -#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0402) -#define TSDB_CODE_DND_DNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0410) -#define TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0411) -#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0420) -#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0421) -#define TSDB_CODE_DND_MNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0422) -#define TSDB_CODE_DND_MNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0423) -#define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0424) -#define TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0430) -#define TSDB_CODE_DND_QNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0431) -#define TSDB_CODE_DND_QNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0432) -#define TSDB_CODE_DND_QNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0433) -#define TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0434) -#define TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0440) -#define TSDB_CODE_DND_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0441) -#define TSDB_CODE_DND_SNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0442) -#define TSDB_CODE_DND_SNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0443) -#define TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0444) -#define TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0450) -#define TSDB_CODE_DND_BNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0451) -#define TSDB_CODE_DND_BNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0452) -#define TSDB_CODE_DND_BNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0453) -#define TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0454) -#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0460) -#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0461) -#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0462) -#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0463) -#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0464) -#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0465) +#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x04A0) +#define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x04A1) +#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x04A2) +#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A3) +#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A4) +#define TSDB_CODE_NODE_PARSE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x04A5) +#define TSDB_CODE_NODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x04A6) +#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A7) +#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A8) +#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x04A9) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x04AA) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) @@ -328,6 +314,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) #define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) +#define TSDB_CODE_VND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0516) +#define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0517) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -353,8 +341,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) -#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0617) +#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x0617) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0618) +#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0619) +#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x0620) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) @@ -456,10 +446,12 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) #define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405) #define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406) +#define TSDB_CODE_CTG_VG_META_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x2407) -//scheduler +//scheduler&qworker #define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) #define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) +#define TSDB_CODE_QW_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x2503) //parser #define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600) @@ -483,6 +475,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612) #define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613) #define TSDB_CODE_PAR_EXPRIE_STATEMENT TAOS_DEF_ERROR_CODE(0, 0x2614) +#define TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2615) #ifdef __cplusplus } diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 943a1f9ecad450f33b4f0e0eb9ef28fae60bd813..838b175a287f3b1ae408f5be549a1fa8082f7b28 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -351,7 +351,7 @@ static FORCE_INLINE void *taosDecodeString(const void *buf, char **value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); - *value = (char *)malloc((size_t)size + 1); + *value = (char *)taosMemoryMalloc((size_t)size + 1); if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)size); @@ -386,7 +386,7 @@ static FORCE_INLINE int32_t taosEncodeBinary(void **buf, const void *value, int3 } static FORCE_INLINE void *taosDecodeBinary(const void *buf, void **value, int32_t valueLen) { - *value = malloc((size_t)valueLen); + *value = taosMemoryMalloc((size_t)valueLen); if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)valueLen); diff --git a/include/util/tdef.h b/include/util/tdef.h index 47fc61947386f5c73b2c80afb999a4d57981a1df..655deb4625e38ba8cdfe4bb558ce7806ab53a72c 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -99,7 +99,7 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_INS_TABLE_MNODES "mnodes" #define TSDB_INS_TABLE_MODULES "modules" #define TSDB_INS_TABLE_QNODES "qnodes" -#define TSDB_INS_TABLE_USER_DATABASE "user_database" +#define TSDB_INS_TABLE_USER_DATABASES "user_databases" #define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions" #define TSDB_INS_TABLE_USER_INDEXES "user_indexes" #define TSDB_INS_TABLE_USER_STABLES "user_stables" @@ -109,6 +109,8 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_INS_TABLE_USER_USERS "user_users" #define TSDB_INS_TABLE_VGROUPS "vgroups" +#define TSDB_INS_USER_STABLES_DBNAME_COLID 2 + #define TSDB_TICK_PER_SECOND(precision) \ ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \ : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) @@ -453,6 +455,10 @@ enum { SND_WORKER_TYPE__UNIQUE, }; +#define MND_VGID -1 +#define QND_VGID 1 +#define VND_VGID 0 + #ifdef __cplusplus } #endif diff --git a/include/util/tencode.h b/include/util/tencode.h index 7e1b624dfb5095536a8c5f170b80af38afc68d52..cdde378b69f7954c168fb633b83139967b37545c 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -406,7 +406,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SCoder* pDecoder, void** val, uin if (tDecodeU64v(pDecoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; - *val = malloc(*len); + *val = taosMemoryMalloc(*len); if (*val == NULL) return -1; memcpy(*val, TD_CODER_CURRENT(pDecoder), *len); diff --git a/include/util/tfreelist.h b/include/util/tfreelist.h index 0a507aeec9b35951e7e646559cf058d5cb05b69a..e9b5ca5fcaa0e26c45cd01d809ce9a4d7def42a9 100644 --- a/include/util/tfreelist.h +++ b/include/util/tfreelist.h @@ -31,7 +31,7 @@ typedef TD_SLIST(SFreeListNode) SFreeList; #define TFL_MALLOC(PTR, TYPE, SIZE, LIST) \ do { \ - void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \ + void *ptr = taosMemoryMalloc((SIZE) + sizeof(struct SFreeListNode)); \ if (ptr) { \ TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ ptr = ((struct SFreeListNode *)ptr)->payload; \ @@ -49,7 +49,7 @@ static FORCE_INLINE void tFreeListClear(SFreeList *pFL) { pNode = TD_SLIST_HEAD(pFL); if (pNode == NULL) break; TD_SLIST_POP(pFL); - free(pNode); + taosMemoryFree(pNode); } } diff --git a/include/util/thash.h b/include/util/thash.h index 57b20c65ee0db89b54d50efae7e090e33101aeec..f2ef445777429b234f1e5e1f46eb6c2282f2a752 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -86,7 +86,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj); * @param size * @return */ -int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size); /** * return the payload data with the specified key diff --git a/include/util/tidpool.h b/include/util/tidpool.h index 8596b439e3f24d684bbabc4c7364f609119d224d..c97f4c5f388ee48f3f86f50b82665f6479c5bb9b 100644 --- a/include/util/tidpool.h +++ b/include/util/tidpool.h @@ -27,7 +27,7 @@ typedef struct { int32_t numOfFree; int32_t freeSlot; bool *freeList; - pthread_mutex_t mutex; + TdThreadMutex mutex; } id_pool_t; void *taosInitIdPool(int32_t maxId); diff --git a/include/util/tjson.h b/include/util/tjson.h index b27f3b93ac5f3f650a3ec829af62d1c08aa2fc00..6b2221f704c768e494ba797913ede2deb5f6a7f4 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -22,6 +22,14 @@ extern "C" { #endif +#define tjsonGetNumberValue(pJson, pName, val) \ + ({ \ + uint64_t _tmp = 0; \ + int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \ + val = _tmp; \ + _code; \ + }) + typedef void SJson; SJson* tjsonCreateObject(); diff --git a/include/util/tlist.h b/include/util/tlist.h index caa642491898243a79509338447452bcd6dd31fa..43833d7ecd84f09643546f3f3fa838edbd1dabf1 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -216,7 +216,7 @@ typedef struct { #define listNEles(l) TD_DLIST_NELES(l) #define listEleSize(l) ((l)->eleSize) #define isListEmpty(l) (TD_DLIST_NELES(l) == 0) -#define listNodeFree(n) free(n) +#define listNodeFree(n) taosMemoryFree(n) void tdListInit(SList *list, int32_t eleSize); void tdListEmpty(SList *list); diff --git a/include/util/tprocess.h b/include/util/tprocess.h new file mode 100644 index 0000000000000000000000000000000000000000..4ce536fd968198ef5be19e888607e647e057853c --- /dev/null +++ b/include/util/tprocess.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_PROCESS_H_ +#define _TD_UTIL_PROCESS_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SProcQueue SProcQueue; +typedef struct SProcObj SProcObj; +typedef void *(*ProcMallocFp)(int32_t contLen); +typedef void *(*ProcFreeFp)(void *pCont); +typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); + +typedef struct { + int32_t childQueueSize; + ProcConsumeFp childConsumeFp; + ProcMallocFp childMallocHeadFp; + ProcFreeFp childFreeHeadFp; + ProcMallocFp childMallocBodyFp; + ProcFreeFp childFreeBodyFp; + int32_t parentQueueSize; + ProcConsumeFp parentConsumeFp; + ProcMallocFp parentdMallocHeadFp; + ProcFreeFp parentFreeHeadFp; + ProcMallocFp parentMallocBodyFp; + ProcFreeFp parentFreeBodyFp; + bool testFlag; + void *pParent; + const char *name; +} SProcCfg; + +SProcObj *taosProcInit(const SProcCfg *pCfg); +void taosProcCleanup(SProcObj *pProc); +int32_t taosProcRun(SProcObj *pProc); +void taosProcStop(SProcObj *pProc); +bool taosProcIsChild(SProcObj *pProc); + +int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); +int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_PROCESS_H_*/ diff --git a/include/util/tqueue.h b/include/util/tqueue.h index d51184edfca2284f314ab9afe490d72430e8ccdf..3bccc7404bfd56dba89e6161e9470e70fd7ae400 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -42,8 +42,14 @@ shall be used to set up the protection. typedef struct STaosQueue STaosQueue; typedef struct STaosQset STaosQset; typedef struct STaosQall STaosQall; -typedef void (*FItem)(void *ahandle, void *pItem); -typedef void (*FItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); +typedef struct { + void *ahandle; + int32_t workerId; + int32_t threadNum; +} SQueueInfo; + +typedef void (*FItem)(SQueueInfo *pInfo, void *pItem); +typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems); STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); @@ -70,10 +76,7 @@ int32_t taosGetQueueNumber(STaosQset *qset); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp); - -int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId); void taosResetQsetThread(STaosQset *qset, void *pItem); - int32_t taosGetQueueItemsNumber(STaosQueue *queue); int32_t taosGetQsetItemsNumber(STaosQset *qset); diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index 64cab08cfe87d430fded815878a5cad6083a4d81..a2382ad541e104430ad7334b458a8d50a5fb5284 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -57,7 +57,7 @@ typedef struct SSkipListNode { * @date 2017/11/12 * the simple version of skip list. * - * for multi-thread safe purpose, we employ pthread_rwlock_t to guarantee to generate + * for multi-thread safe purpose, we employ TdThreadRwlock to guarantee to generate * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance. * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of * this simple version in a multi-thread environment, to achieve higher performance of read/write operations. @@ -106,7 +106,7 @@ typedef struct SSkipList { uint32_t seed; __compar_fn_t comparFn; __sl_key_fn_t keyFn; - pthread_rwlock_t *lock; + TdThreadRwlock *lock; uint16_t len; uint8_t maxLevel; uint8_t flags; diff --git a/include/util/tthread.h b/include/util/tthread.h index 49412069445d0f1cb89631683daa6411ca88879d..2215add06262ea63cbeb447f505c3a9e2bc134de 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -22,9 +22,11 @@ extern "C" { #endif -pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param); -bool taosDestoryThread(pthread_t* pthread); -bool taosThreadRunning(pthread_t* pthread); +TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param); +bool taosDestoryThread(TdThread* pthread); +bool taosThreadRunning(TdThread* pthread); + +typedef void *(*ThreadFp)(void *param); #ifdef __cplusplus } diff --git a/include/util/tworker.h b/include/util/tworker.h index e6f6bc077c69f05498df2536c7de55a488dd8f93..92d474c8857b58cdb8a8680be8eb45de3634e046 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -27,34 +27,35 @@ typedef struct SWWorkerPool SWWorkerPool; typedef struct SQWorker { int32_t id; // worker ID - pthread_t thread; // thread + TdThread thread; // thread SQWorkerPool *pool; -} SQWorker, SFWorker; +} SQWorker; typedef struct SQWorkerPool { - int32_t max; // max number of workers - int32_t min; // min number of workers - int32_t num; // current number of workers - STaosQset *qset; - const char *name; - SQWorker *workers; - pthread_mutex_t mutex; -} SQWorkerPool, SFWorkerPool; + int32_t max; // max number of workers + int32_t min; // min number of workers + int32_t num; // current number of workers + STaosQset *qset; + const char *name; + SQWorker *workers; + TdThreadMutex mutex; +} SQWorkerPool; typedef struct SWWorker { int32_t id; // worker id - pthread_t thread; // thread + TdThread thread; // thread STaosQall *qall; STaosQset *qset; // queue set SWWorkerPool *pool; } SWWorker; typedef struct SWWorkerPool { - int32_t max; // max number of workers - int32_t nextId; // from 0 to max-1, cyclic - const char *name; - SWWorker *workers; - pthread_mutex_t mutex; + int32_t max; // max number of workers + int32_t num; + int32_t nextId; // from 0 to max-1, cyclic + const char *name; + SWWorker *workers; + TdThreadMutex mutex; } SWWorkerPool; int32_t tQWorkerInit(SQWorkerPool *pool); @@ -62,16 +63,43 @@ void tQWorkerCleanup(SQWorkerPool *pool); STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp); void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue); -int32_t tFWorkerInit(SFWorkerPool *pool); -void tFWorkerCleanup(SFWorkerPool *pool); -STaosQueue *tFWorkerAllocQueue(SFWorkerPool *pool, void *ahandle, FItem fp); -void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue); - int32_t tWWorkerInit(SWWorkerPool *pool); void tWWorkerCleanup(SWWorkerPool *pool); STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp); void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue); +typedef struct { + const char *name; + int32_t minNum; + int32_t maxNum; + FItem fp; + void *param; +} SSingleWorkerCfg; + +typedef struct { + const char *name; + STaosQueue *queue; + SQWorkerPool pool; +} SSingleWorker; + +typedef struct { + const char *name; + int32_t maxNum; + FItems fp; + void *param; +} SMultiWorkerCfg; + +typedef struct { + const char *name; + STaosQueue *queue; + SWWorkerPool pool; +} SMultiWorker; + +int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg); +void tSingleWorkerCleanup(SSingleWorker *pWorker); +int32_t tMultiWorkerInit(SMultiWorker *pWorker, const SMultiWorkerCfg *pCfg); +void tMultiWorkerCleanup(SMultiWorker *pWorker); + #ifdef __cplusplus } #endif diff --git a/include/util/types.h b/include/util/types.h index 981c457fc10598d7109c44845939364c428f7a66..d48995418e530646e8ef540bfb0630fc90cd9636 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -84,6 +84,8 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) +#define NCHAR_WIDTH_TO_BYTES(n) ((n) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE) + typedef int32_t VarDataOffsetT; typedef struct tstr { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index e341729a8fcf9d5a398d7f59fb913f88e18a1936..7d7e51bc27fa2fc858a6a346d110a345b060c73d 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -37,9 +37,8 @@ extern "C" { #define CHECK_CODE_GOTO(expr, label) \ do { \ - int32_t code = expr; \ + code = expr; \ if (TSDB_CODE_SUCCESS != code) { \ - terrno = code; \ goto label; \ } \ } while (0) @@ -77,8 +76,8 @@ typedef struct { int8_t inited; // ctl int8_t threadStop; - pthread_t thread; - pthread_mutex_t lock; // used when app init and cleanup + TdThread thread; + TdThreadMutex lock; // used when app init and cleanup SArray* appHbMgrs; // SArray one for each cluster FHbReqHandle reqHandle[HEARTBEAT_TYPE_MAX]; FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; @@ -125,7 +124,7 @@ typedef struct SAppInfo { int32_t pid; int32_t numOfThreads; SHashObj* pInstMap; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SAppInfo; typedef struct STscObj { @@ -137,23 +136,33 @@ typedef struct STscObj { uint32_t connId; int32_t connType; uint64_t id; // ref ID returned by taosAddRef - pthread_mutex_t mutex; // used to protect the operation on db + TdThreadMutex mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; } STscObj; +typedef struct SResultColumn { + union { + char* nullbitmap; // bitmap, one bit for each item in the list + int32_t* offset; + }; + char* pData; +} SResultColumn; + typedef struct SReqResultInfo { - const char* pRspMsg; - const char* pData; - TAOS_FIELD* fields; - uint32_t numOfCols; - int32_t* length; - TAOS_ROW row; - char** pCol; - uint32_t numOfRows; - uint64_t totalRows; - uint32_t current; - bool completed; + const char* pRspMsg; + const char* pData; + TAOS_FIELD* fields; + uint32_t numOfCols; + int32_t* length; + char** convertBuf; + TAOS_ROW row; + SResultColumn* pCol; + uint32_t numOfRows; + uint64_t totalRows; + uint32_t current; + bool completed; + int32_t payloadLen; } SReqResultInfo; typedef struct SShowReqInfo { @@ -186,6 +195,8 @@ typedef struct SRequestObj { char* msgBuf; void* pInfo; // sql parse info, generated by parser module int32_t code; + SArray* dbList; + SArray* tableList; SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -226,11 +237,11 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, void* doFetchRow(SRequestObj* pRequest); -void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); -int32_t parseSql(SRequestObj* pRequest, bool streamQuery, SQuery** pQuery); +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery); int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); // --- heartbeat diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 2233742625e6e733f184bd2c4e7eac376afd9139..d9dcadbff0995edb2289d1cd03c45ec48aafb038 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -33,7 +33,7 @@ SAppInfo appInfo; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; +static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj *pRequest) { @@ -48,8 +48,8 @@ static void registerRequest(SRequestObj *pRequest) { if (pTscObj->pAppInfo) { SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary; - int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1); - int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1); + int32_t total = atomic_add_fetch_64(&pSummary->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_64(&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); @@ -62,7 +62,7 @@ static void deregisterRequest(SRequestObj *pRequest) { STscObj * pTscObj = pRequest->pTscObj; SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; - int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); + int32_t currentInst = atomic_sub_fetch_64(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampMs() - pRequest->metric.start; @@ -90,7 +90,6 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; - rpcInit.pfp = persistConnForSpecificMsg; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; @@ -115,12 +114,12 @@ void destroyTscObj(void *pObj) { hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); - pthread_mutex_destroy(&pTscObj->mutex); - tfree(pTscObj); + taosThreadMutexDestroy(&pTscObj->mutex); + taosMemoryFreeClear(pTscObj); } void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { - STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); + STscObj *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -134,7 +133,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI tstrncpy(pObj->db, db, tListLen(pObj->db)); } - pthread_mutex_init(&pObj->mutex, NULL); + taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); tscDebug("connObj created, 0x%" PRIx64, pObj->id); @@ -144,7 +143,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); - SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); if (NULL == pRequest) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -157,7 +156,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty pRequest->type = type; pRequest->pTscObj = pObj; pRequest->body.fp = fp; // not used it yet - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); @@ -165,11 +164,18 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty } static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { - tfree(pResInfo->pRspMsg); - tfree(pResInfo->length); - tfree(pResInfo->row); - tfree(pResInfo->pCol); - tfree(pResInfo->fields); + taosMemoryFreeClear(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->length); + taosMemoryFreeClear(pResInfo->row); + taosMemoryFreeClear(pResInfo->pCol); + taosMemoryFreeClear(pResInfo->fields); + + if (pResInfo->convertBuf != NULL) { + for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { + taosMemoryFreeClear(pResInfo->convertBuf[i]); + } + taosMemoryFreeClear(pResInfo->convertBuf); + } } static void doDestroyRequest(void *p) { @@ -178,20 +184,24 @@ static void doDestroyRequest(void *p) { assert(RID_VALID(pRequest->self)); - tfree(pRequest->msgBuf); - tfree(pRequest->sqlstr); - tfree(pRequest->pInfo); - tfree(pRequest->pDb); + taosMemoryFreeClear(pRequest->msgBuf); + taosMemoryFreeClear(pRequest->sqlstr); + taosMemoryFreeClear(pRequest->pInfo); + taosMemoryFreeClear(pRequest->pDb); doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryPlan(pRequest->body.pDag); + if (pRequest->body.queryJob != 0) { + schedulerFreeJob(pRequest->body.queryJob); + } + if (pRequest->body.showInfo.pArray != NULL) { taosArrayDestroy(pRequest->body.showInfo.pArray); } deregisterRequest(pRequest); - tfree(pRequest); + taosMemoryFreeClear(pRequest); } void destroyRequest(SRequestObj *pRequest) { @@ -243,7 +253,7 @@ void taos_init_imp(void) { // transDestroyBuffer(&conn->readBuf); taosGetAppName(appInfo.appName, NULL); - pthread_mutex_init(&appInfo.mutex, NULL); + taosThreadMutexInit(&appInfo.mutex, NULL); appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); @@ -252,7 +262,7 @@ void taos_init_imp(void) { } int taos_init() { - pthread_once(&tscinit, taos_init_imp); + taosThreadOnce(&tscinit, taos_init_imp); return tscInitRes; } @@ -350,7 +360,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); } - free(charset); + taosMemoryFree(charset); } else { // it may be windows system tscInfo("charset remains:%s", tsCharset); } @@ -508,9 +518,9 @@ static setConfRet taos_set_config_imp(const char *config){ } setConfRet taos_set_config(const char *config){ - pthread_mutex_lock(&setConfMutex); + taosThreadMutexLock(&setConfMutex); setConfRet ret = taos_set_config_imp(config); - pthread_mutex_unlock(&setConfMutex); + taosThreadMutexUnlock(&setConfMutex); return ret; } #endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 71634447196f640ee8fc5d4358876e15000c5534..832a415efac6058340b22658e332bf871af7a3ec 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -166,7 +166,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { static int32_t emptyRspNum = 0; if (code != 0) { - tfree(param); + taosMemoryFreeClear(param); return -1; } @@ -179,12 +179,12 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL || NULL == *pInst) { tscError("cluster not exist, key:%s", key); - tfree(param); + taosMemoryFreeClear(param); tFreeClientHbBatchRsp(&pRsp); return -1; } - tfree(param); + taosMemoryFreeClear(param); if (rspNum) { tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, @@ -317,7 +317,7 @@ void hbFreeReq(void *req) { } SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { - SClientHbBatchReq *pBatchReq = calloc(1, sizeof(SClientHbBatchReq)); + SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -346,7 +346,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { if (code) { taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq); - tfree(pBatchReq); + taosMemoryFreeClear(pBatchReq); } return pBatchReq; @@ -372,7 +372,7 @@ static void *hbThreadFunc(void *param) { break; } - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { @@ -387,7 +387,7 @@ static void *hbThreadFunc(void *param) { continue; } int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); - void *buf = malloc(tlen); + void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq, false); @@ -396,13 +396,13 @@ static void *hbThreadFunc(void *param) { } tSerializeSClientHbBatchReq(buf, tlen, pReq); - SMsgSendInfo *pInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq, false); hbClearReqInfo(pAppHbMgr); - free(buf); + taosMemoryFree(buf); break; } pInfo->fp = hbAsyncCallBack; @@ -423,7 +423,7 @@ static void *hbThreadFunc(void *param) { atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); taosMsleep(HEARTBEAT_INTERVAL); } @@ -431,15 +431,15 @@ static void *hbThreadFunc(void *param) { } static int32_t hbCreateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); -// if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { +// if (taosThreadCreate(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { // terrno = TAOS_SYSTEM_ERROR(errno); // return -1; // } -// pthread_attr_destroy(&thAttr); +// taosThreadAttrDestroy(&thAttr); return 0; } @@ -458,7 +458,7 @@ static void hbStopThread() { SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { hbMgrInit(); - SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr)); + SAppHbMgr *pAppHbMgr = taosMemoryMalloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -478,7 +478,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { if (pAppHbMgr->activeInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(pAppHbMgr); + taosMemoryFree(pAppHbMgr); return NULL; } @@ -488,19 +488,19 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { if (pAppHbMgr->connInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(pAppHbMgr); + taosMemoryFree(pAppHbMgr); return NULL; } - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); return pAppHbMgr; } void appHbMgrCleanup(void) { - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { @@ -511,7 +511,7 @@ void appHbMgrCleanup(void) { pTarget->connInfo = NULL; } - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); } int hbMgrInit() { @@ -520,7 +520,7 @@ int hbMgrInit() { if (old == 1) return 0; clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); - pthread_mutex_init(&clientHbMgr.lock, NULL); + taosThreadMutexInit(&clientHbMgr.lock, NULL); // init handle funcs hbMgrInitHandle(); @@ -539,10 +539,10 @@ void hbMgrCleanUp() { int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); if (old == 0) return; - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); appHbMgrCleanup(); taosArrayDestroy(clientHbMgr.appHbMgrs); - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); clientHbMgr.appHbMgrs = NULL; #endif @@ -580,7 +580,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3 switch (hbType) { case HEARTBEAT_TYPE_QUERY: { - int64_t *pClusterId = malloc(sizeof(int64_t)); + int64_t *pClusterId = taosMemoryMalloc(sizeof(int64_t)); *pClusterId = clusterId; info.param = pClusterId; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index aa94ed42fd1838ce34200d9195d8e9cdb0d48c7c..6e65a4267fe5ded075583b9d0c5e75235ed53bc6 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -12,7 +12,7 @@ static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); -static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); +static int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { @@ -95,12 +95,12 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, char* key = getClusterKey(user, secretEncrypt, ip, port); SAppInstInfo** pInst = NULL; - pthread_mutex_lock(&appInfo.mutex); + taosThreadMutexLock(&appInfo.mutex); pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); SAppInstInfo* p = NULL; if (pInst == NULL) { - p = calloc(1, sizeof(struct SAppInstInfo)); + p = taosMemoryCalloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); p->pAppHbMgr = appHbMgrInit(p, key); @@ -109,9 +109,9 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, pInst = &p; } - pthread_mutex_unlock(&appInfo.mutex); + taosThreadMutexUnlock(&appInfo.mutex); - tfree(key); + taosMemoryFreeClear(key); return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst); } @@ -122,7 +122,7 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* return TSDB_CODE_TSC_OUT_OF_MEMORY; } - (*pRequest)->sqlstr = malloc(sqlLen + 1); + (*pRequest)->sqlstr = taosMemoryMalloc(sqlLen + 1); if ((*pRequest)->sqlstr == NULL) { tscError("0x%" PRIx64 " failed to prepare sql string buffer", (*pRequest)->self); (*pRequest)->msgBuf = strdup("failed to prepare sql string buffer"); @@ -137,14 +137,14 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* return TSDB_CODE_SUCCESS; } -int32_t parseSql(SRequestObj* pRequest, bool streamQuery, SQuery** pQuery) { +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { STscObj* pTscObj = pRequest->pTscObj; SParseContext cxt = { .requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = pRequest->pDb, - .streamQuery = streamQuery, + .topicQuery = topicQuery, .pSql = pRequest->sqlstr, .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, @@ -159,8 +159,12 @@ int32_t parseSql(SRequestObj* pRequest, bool streamQuery, SQuery** pQuery) { } code = qParseQuerySql(&cxt, pQuery); - if (TSDB_CODE_SUCCESS == code && ((*pQuery)->haveResultSet)) { - setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); + if (TSDB_CODE_SUCCESS == code) { + if ((*pQuery)->haveResultSet) { + setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); + } + TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*); + TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*); } return code; @@ -191,15 +195,25 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) { pRequest->type = pQuery->msgType; - SPlanContext cxt = { .queryId = pRequest->requestId, .pAstRoot = pQuery->pRoot, .acctId = pRequest->pTscObj->acctId }; - return qCreateQueryPlan(&cxt, pPlan, pNodeList); + SPlanContext cxt = { + .queryId = pRequest->requestId, + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite + }; + int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList); + if (code != 0) { + return code; + } + return code; } void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { assert(pSchema != NULL && numOfCols > 0); pResInfo->numOfCols = numOfCols; - pResInfo->fields = calloc(numOfCols, sizeof(pSchema[0])); + pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(pSchema[0])); for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { pResInfo->fields[i].bytes = pSchema[i].bytes; @@ -219,6 +233,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } pRequest->code = code; + terrno = code; return pRequest->code; } @@ -231,22 +246,16 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } pRequest->code = res.code; + terrno = res.code; return pRequest->code; } -TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { - STscObj* pTscObj = (STscObj*)taos; - if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - return NULL; - } - +SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; + int32_t code = 0; SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); - terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return); @@ -255,19 +264,91 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { } else { CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); - pRequest->code = terrno; } _return: taosArrayDestroy(pNodeList); qDestroyQuery(pQuery); - if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { + if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } return pRequest; } +int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { + SCatalog *pCatalog = NULL; + int32_t code = 0; + int32_t dbNum = taosArrayGetSize(pRequest->dbList); + int32_t tblNum = taosArrayGetSize(pRequest->tableList); + + if (dbNum <= 0 && tblNum <= 0) { + return TSDB_CODE_QRY_APP_ERROR; + } + + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + SEpSet epset = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + for (int32_t i = 0; i < dbNum; ++i) { + char *dbFName = taosArrayGet(pRequest->dbList, i); + + code = catalogRefreshDBVgInfo(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, dbFName); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + for (int32_t i = 0; i < tblNum; ++i) { + SName *tableName = taosArrayGet(pRequest->tableList, i); + + code = catalogRefreshTableMeta(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, tableName, -1); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + + +SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { + SRequestObj* pRequest = NULL; + int32_t retryNum = 0; + int32_t code = 0; + + while (retryNum++ < REQUEST_MAX_TRY_TIMES) { + pRequest = execQueryImpl(pTscObj, sql, sqlLen); + if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { + break; + } + + code = refreshMeta(pTscObj, pRequest); + if (code) { + pRequest->code = code; + break; + } + + destroyRequest(pRequest); + } + + return pRequest; +} + +TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { + STscObj* pTscObj = (STscObj*)taos; + if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + return NULL; + } + + return execQuery(pTscObj, sql, sqlLen); +} + int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { pEpSet->version = 0; @@ -343,7 +424,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t } static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -363,14 +444,14 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { if (db != NULL) { tstrncpy(connectReq.db, db, sizeof(connectReq.db)); } - tfree(db); + taosMemoryFreeClear(db); connectReq.pid = htonl(appInfo.pid); connectReq.startTime = htobe64(appInfo.startTime); tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); - void* pReq = malloc(contLen); + void* pReq = taosMemoryMalloc(contLen); tSerializeSConnectReq(pReq, contLen, &connectReq); pMsgSendInfo->msgInfo.len = contLen; @@ -380,11 +461,11 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { - return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; + return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || msgType == TDMT_VND_QUERY_HEARTBEAT_RSP; } void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle; @@ -395,7 +476,6 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { assert(pRequest->self == pSendInfo->requestObjRefId); pRequest->metric.rsp = taosGetTimestampMs(); - pRequest->code = pMsg->code; STscObj* pTscObj = pRequest->pTscObj; if (pEpSet) { @@ -423,7 +503,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SDataBuf buf = {.len = pMsg->contLen, .pData = NULL, .handle = pMsg->handle}; if (pMsg->contLen > 0) { - buf.pData = calloc(1, pMsg->contLen); + buf.pData = taosMemoryCalloc(1, pMsg->contLen); if (buf.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY; @@ -479,13 +559,16 @@ void* doFetchRow(SRequestObj* pRequest) { } SReqResultInfo* pResInfo = &pRequest->body.resInfo; - int32_t code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData); - if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; + pRequest->code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData); + if (pRequest->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData); + if (pRequest->code != TSDB_CODE_SUCCESS) { return NULL; } - setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData); tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); @@ -512,7 +595,7 @@ void* doFetchRow(SRequestObj* pRequest) { } SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); - SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq)); pShowReq->head.vgId = htonl(pVgroupInfo->vgId); pRequest->body.requestMsg.len = sizeof(SVShowTablesReq); @@ -552,10 +635,35 @@ void* doFetchRow(SRequestObj* pRequest) { _return: for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { - pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; - if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { - pResultInfo->length[i] = varDataLen(pResultInfo->row[i]); - pResultInfo->row[i] = varDataVal(pResultInfo->row[i]); + SResultColumn* pCol = &pResultInfo->pCol[i]; + + int32_t type = pResultInfo->fields[i].type; + int32_t bytes = pResultInfo->fields[i].bytes; + + if (IS_VAR_DATA_TYPE(type)) { + if (pCol->offset[pResultInfo->current] != -1) { + char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData; + + pResultInfo->length[i] = varDataLen(pStart); + pResultInfo->row[i] = varDataVal(pStart); + + if (type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(pResultInfo->convertBuf[i])); + ASSERT(len <= bytes); + + pResultInfo->row[i] = varDataVal(pResultInfo->convertBuf[i]); + varDataSetLen(pResultInfo->convertBuf[i], len); + pResultInfo->length[i] = len; + } + } else { + pResultInfo->row[i] = NULL; + } + } else { + if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) { + pResultInfo->row[i] = pResultInfo->pCol[i].pData + bytes * pResultInfo->current; + } else { + pResultInfo->row[i] = NULL; + } } } @@ -563,60 +671,91 @@ _return: return pResultInfo->row; } -static void doPrepareResPtr(SReqResultInfo* pResInfo) { +static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { if (pResInfo->row == NULL) { - pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); + pResInfo->row = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = taosMemoryCalloc(pResInfo->numOfCols, sizeof(SResultColumn)); + pResInfo->length = taosMemoryCalloc(pResInfo->numOfCols, sizeof(int32_t)); + pResInfo->convertBuf = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); + + if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL || pResInfo->convertBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for(int32_t i = 0; i < pResInfo->numOfCols; ++i) { + if(pResInfo->fields[i].type == TSDB_DATA_TYPE_NCHAR) { + pResInfo->convertBuf[i] = taosMemoryCalloc(1, NCHAR_WIDTH_TO_BYTES(pResInfo->fields[i].bytes)); + } + } } + + return TSDB_CODE_SUCCESS; } -void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); if (numOfRows == 0) { - return; + return TSDB_CODE_SUCCESS; } - // todo check for the failure of malloc - doPrepareResPtr(pResultInfo); + int32_t code = doPrepareResPtr(pResultInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int32_t offset = 0; + int32_t* colLength = (int32_t*)pResultInfo->pData; + char* pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { + colLength[i] = htonl(colLength[i]); + + if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { + pResultInfo->pCol[i].offset = (int32_t*)pStart; + pStart += numOfRows * sizeof(int32_t); + } else { + pResultInfo->pCol[i].nullbitmap = pStart; + pStart += BitmapLen(pResultInfo->numOfRows); + } + + pResultInfo->pCol[i].pData = pStart; pResultInfo->length[i] = pResultInfo->fields[i].bytes; - pResultInfo->row[i] = (char*)(pResultInfo->pData + offset * pResultInfo->numOfRows); - pResultInfo->pCol[i] = pResultInfo->row[i]; - offset += pResultInfo->fields[i].bytes; + pResultInfo->row[i] = pResultInfo->pCol[i].pData; + + pStart += colLength[i]; } + + return TSDB_CODE_SUCCESS; } char* getDbOfConnection(STscObj* pObj) { char* p = NULL; - pthread_mutex_lock(&pObj->mutex); + taosThreadMutexLock(&pObj->mutex); size_t len = strlen(pObj->db); if (len > 0) { p = strndup(pObj->db, tListLen(pObj->db)); } - pthread_mutex_unlock(&pObj->mutex); + taosThreadMutexUnlock(&pObj->mutex); return p; } void setConnectionDB(STscObj* pTscObj, const char* db) { assert(db != NULL && pTscObj != NULL); - pthread_mutex_lock(&pTscObj->mutex); + taosThreadMutexLock(&pTscObj->mutex); tstrncpy(pTscObj->db, db, tListLen(pTscObj->db)); - pthread_mutex_unlock(&pTscObj->mutex); + taosThreadMutexUnlock(&pTscObj->mutex); } -void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { assert(pResultInfo != NULL && pRsp != NULL); - pResultInfo->pRspMsg = (const char*)pRsp; - pResultInfo->pData = (void*)pRsp->data; - pResultInfo->numOfRows = htonl(pRsp->numOfRows); - pResultInfo->current = 0; - pResultInfo->completed = (pRsp->completed == 1); + pResultInfo->pRspMsg = (const char*)pRsp; + pResultInfo->pData = (void*)pRsp->data; + pResultInfo->numOfRows = htonl(pRsp->numOfRows); + pResultInfo->current = 0; + pResultInfo->completed = (pRsp->completed == 1); + pResultInfo->payloadLen = htonl(pRsp->compLen); + // TODO handle the compressed case pResultInfo->totalRows += pResultInfo->numOfRows; - setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); + return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 010f4e6c12e6a24d7882d82ce731daa13a401d32..43143917436435d735a9cdc733902e60f20457a0 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -32,7 +32,7 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; setErrno(pRequest, code); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return code; } @@ -40,7 +40,7 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { - free(pMsg->pData); + taosMemoryFree(pMsg->pData); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; @@ -77,13 +77,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, pTscObj->pAppInfo->numOfConns); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return 0; } SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; @@ -96,13 +96,13 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { retrieveReq.showId = pRequest->body.showInfo.execId; int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); - void* pReq = malloc(contLen); + void* pReq = taosMemoryMalloc(contLen); tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); pMsgSendInfo->msgInfo.pData = pReq; pMsgSendInfo->msgInfo.len = contLen; pMsgSendInfo->msgInfo.handle = NULL; } else { - SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); + SVShowTablesFetchReq* pFetchMsg = taosMemoryCalloc(1, sizeof(SVShowTablesFetchReq)); if (pFetchMsg == NULL) { return NULL; } @@ -135,12 +135,12 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp); STableMetaRsp *pMetaMsg = &showRsp.tableMeta; - tfree(pRequest->body.resInfo.pRspMsg); + taosMemoryFreeClear(pRequest->body.resInfo.pRspMsg); pRequest->body.resInfo.pRspMsg = pMsg->pData; SReqResultInfo* pResInfo = &pRequest->body.resInfo; if (pResInfo->fields == NULL) { - TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); + TAOS_FIELD* pFields = taosMemoryCalloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { SSchema* pSchema = &pMetaMsg->pSchemas[i]; tstrncpy(pFields[i].name, pSchema->name, tListLen(pFields[i].name)); @@ -171,7 +171,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj *pRequest = param; SReqResultInfo *pResInfo = &pRequest->body.resInfo; - tfree(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -204,7 +204,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; SReqResultInfo* pResInfo = &pRequest->body.resInfo; - tfree(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -237,8 +237,12 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { // todo rsp with the vnode id list SRequestObj* pRequest = param; - free(pMsg->pData); + taosMemoryFree(pMsg->pData); + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + } tsem_post(&pRequest->body.rspSem); + return code; } int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { @@ -262,7 +266,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { } if (code != TSDB_CODE_SUCCESS) { - free(pMsg->pData); + taosMemoryFree(pMsg->pData); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; @@ -280,7 +284,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (code != 0) { terrno = code; if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash); - tfree(output.dbVgroup); + taosMemoryFreeClear(output.dbVgroup); tscError("failed to build use db output since %s", terrstr()); } else { @@ -300,7 +304,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { tNameGetDbName(&name, db); setConnectionDB(pRequest->pTscObj, db); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return 0; } @@ -309,7 +313,7 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { assert(pMsg != NULL && param != NULL); SRequestObj* pRequest = param; - free(pMsg->pData); + taosMemoryFree(pMsg->pData); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 2aebd67c56c53d98bffe515313a5567840ee2d7e..52d5400b0bffe8c6f352a8bd2731cccdbb8f13a0 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -136,14 +136,14 @@ typedef struct { } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { - tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); + tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; return conf; } void tmq_conf_destroy(tmq_conf_t* conf) { - if (conf) free(conf); + if (conf) taosMemoryFree(conf); } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { @@ -184,7 +184,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } tmq_list_t* tmq_list_new() { - tmq_list_t* ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); + tmq_list_t* ptr = taosMemoryMalloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); if (ptr == NULL) { return ptr; } @@ -254,7 +254,7 @@ tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { } tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - tmq_t* pTmq = calloc(sizeof(tmq_t), 1); + tmq_t* pTmq = taosMemoryCalloc(sizeof(tmq_t), 1); if (pTmq == NULL) { return NULL; } @@ -317,7 +317,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); tEncodeSMqCMCommitOffsetReq(&encoder, &req); int32_t tlen = encoder.pos; - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { tCoderClear(&encoder); return -1; @@ -333,7 +333,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tscError("failed to malloc request"); } - SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); if (pParam == NULL) { return -1; } @@ -361,7 +361,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in } tsem_destroy(&pParam->rspSem); - free(pParam); + taosMemoryFree(pParam); if (pArray) { taosArrayDestroy(pArray); @@ -394,7 +394,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); tNameFromString(&name, topicName, T_NAME_TABLE); - char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); + char* topicFname = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFname == NULL) { goto _return; } @@ -405,11 +405,11 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); taosArrayPush(tmq->clientTopics, &topic); taosArrayPush(req.topicNames, &topicFname); - free(dbName); + taosMemoryFree(dbName); } int tlen = tSerializeSCMSubscribeReq(NULL, &req); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { goto _return; } @@ -456,11 +456,100 @@ _return: void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } +TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) { + STscObj* pTscObj = (STscObj*)taos; + SRequestObj* pRequest = NULL; + SQuery* pQueryNode = NULL; + char* astStr = NULL; + int32_t sqlLen; + + terrno = TSDB_CODE_SUCCESS; + if (taos == NULL || streamName == NULL || sql == NULL) { + tscError("invalid parameters for creating stream, connObj:%p, stream name:%s, sql:%s", taos, streamName, sql); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + sqlLen = strlen(sql); + + if (strlen(tbName) >= TSDB_TABLE_NAME_LEN) { + tscError("output tb name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + goto _return; + } + + tscDebug("start to create stream: %s", streamName); + + int32_t code = 0; + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode), _return); + + // todo check for invalid sql statement and return with error code + + CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return); + + /*printf("%s\n", pStr);*/ + + SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T}; + strcpy(name.dbname, pRequest->pDb); + strcpy(name.tname, streamName); + + SCMCreateStreamReq req = { + .igExists = 1, + .ast = astStr, + .sql = (char*)sql, + }; + tNameExtractFullName(&name, req.name); + strcpy(req.outputSTbName, tbName); + + int tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req); + void* buf = taosMemoryMalloc(tlen); + if (buf == NULL) { + goto _return; + } + + tSerializeSCMCreateStreamReq(buf, tlen, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest->body.requestMsg = (SDataBuf){ + .pData = buf, + .len = tlen, + .handle = NULL, + }; + pRequest->type = TDMT_MND_CREATE_STREAM; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(&pRequest->body.rspSem); + +_return: + taosMemoryFreeClear(astStr); + qDestroyQuery(pQueryNode); + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ + + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { + pRequest->code = terrno; + } + + return pRequest; +} + TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { STscObj* pTscObj = (STscObj*)taos; SRequestObj* pRequest = NULL; SQuery* pQueryNode = NULL; - char* pStr = NULL; + char* astStr = NULL; terrno = TSDB_CODE_SUCCESS; if (taos == NULL || topicName == NULL || sql == NULL) { @@ -481,30 +570,31 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i goto _return; } - tscDebug("start to create topic, %s", topicName); + tscDebug("start to create topic: %s", topicName); + int32_t code = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return); // todo check for invalid sql statement and return with error code - CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &pStr, NULL), _return); + CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return); /*printf("%s\n", pStr);*/ - SName name = { .acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T }; + SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T}; strcpy(name.dbname, pRequest->pDb); strcpy(name.tname, topicName); SCMCreateTopicReq req = { .igExists = 1, - .ast = (char*)pStr, + .ast = astStr, .sql = (char*)sql, }; tNameExtractFullName(&name, req.name); int tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { goto _return; } @@ -512,7 +602,11 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tSerializeSCMCreateTopicReq(buf, tlen, &req); /*printf("formatted: %s\n", dagStr);*/ - pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; + pRequest->body.requestMsg = (SDataBuf){ + .pData = buf, + .len = tlen, + .handle = NULL, + }; pRequest->type = TDMT_MND_CREATE_TOPIC; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); @@ -524,6 +618,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tsem_wait(&pRequest->body.rspSem); _return: + taosMemoryFreeClear(astStr); qDestroyQuery(pQueryNode); /*if (sendInfo != NULL) {*/ /*destroySendMsgInfo(sendInfo);*/ @@ -588,7 +683,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; - SMqPollRsp* pRsp = &tmq_message->consumeRsp; + SMqPollRsp* pRsp = &tmq_message->msg; return pRsp->skipLogNum; } @@ -597,15 +692,15 @@ void tmqShowMsg(tmq_message_t* tmq_message) { static bool noPrintSchema; char pBuf[128]; - SMqPollRsp* pRsp = &tmq_message->consumeRsp; - int32_t colNum = pRsp->schemas->nCols; + SMqPollRsp* pRsp = &tmq_message->msg; + int32_t colNum = pRsp->schema->nCols; if (!noPrintSchema) { printf("|"); for (int32_t i = 0; i < colNum; i++) { if (i == 0) - printf(" %25s |", pRsp->schemas->pSchema[i].name); + printf(" %25s |", pRsp->schema->pSchema[i].name); else - printf(" %15s |", pRsp->schemas->pSchema[i].name); + printf(" %15s |", pRsp->schema->pSchema[i].name); } printf("\n"); printf("===============================================\n"); @@ -642,7 +737,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqClientVg* pVg = pParam->pVg; tmq_t* tmq = pParam->tmq; if (code != 0) { - printf("msg discard\n"); + printf("msg discard %x\n", code); goto WRITE_QUEUE_FAIL; } @@ -662,7 +757,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { #if 0 if (pParam->sync == 1) { - /**pParam->msg = malloc(sizeof(tmq_message_t));*/ + /**pParam->msg = taosMemoryMalloc(sizeof(tmq_message_t));*/ *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t)); if (*pParam->msg) { memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead)); @@ -679,25 +774,25 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ + /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/ tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); if (pRsp == NULL) { goto WRITE_QUEUE_FAIL; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); - pRsp->curBlock = 0; - pRsp->curRow = 0; + tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg); + pRsp->iter.curBlock = 0; + pRsp->iter.curRow = 0; // TODO: alloc mem /*pRsp->*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ - if (pRsp->consumeRsp.numOfTopics == 0) { + if (pRsp->msg.numOfTopics == 0) { /*printf("no data\n");*/ taosFreeQitem(pRsp); goto WRITE_QUEUE_FAIL; } - pRsp->extra = pParam->pVg; + pRsp->vg = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); atomic_add_fetch_32(&tmq->readyRequest, 1); tsem_post(&tmq->rspSem); @@ -767,14 +862,14 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { } tDeleteSMqCMGetSubEpRsp(&rsp); } else { - tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); + SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp)); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; goto END; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp); + tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp); taosWriteQitem(tmq->mqueue, pRsp); tsem_post(&tmq->rspSem); @@ -789,7 +884,7 @@ END: int32_t tmqAskEp(tmq_t* tmq, bool sync) { int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* req = malloc(tlen); + SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen); if (req == NULL) { tscError("failed to malloc get subscribe ep buf"); return -1; @@ -798,21 +893,21 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { req->epoch = htonl(tmq->epoch); strcpy(req->cgroup, tmq->groupId); - SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); + SMqAskEpCbParam* pParam = taosMemoryMalloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { tscError("failed to malloc subscribe param"); - free(req); + taosMemoryFree(req); return -1; } pParam->tmq = tmq; pParam->sync = sync; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { tsem_destroy(&pParam->rspSem); - free(pParam); - free(req); + taosMemoryFree(pParam); + taosMemoryFree(req); return -1; } @@ -872,7 +967,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo reqOffset = tmq->resetOffsetCfg; } - SMqPollReq* pReq = malloc(sizeof(SMqPollReq)); + SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; } @@ -890,6 +985,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo return pReq; } +#if 0 tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { tmq_message_t* msg = NULL; for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { @@ -907,7 +1003,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { return NULL; } - SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // TODO: out of mem @@ -920,7 +1016,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { pParam->msg = &msg; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { return NULL; } @@ -957,6 +1053,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { } return NULL; } +#endif int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { /*printf("call poll\n");*/ @@ -974,9 +1071,9 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { tsem_post(&tmq->rspSem); return -1; } - SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { - free(pReq); + taosMemoryFree(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; @@ -986,10 +1083,10 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { pParam->epoch = tmq->epoch; pParam->sync = 0; - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { - free(pReq); - free(pParam); + taosMemoryFree(pReq); + taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; @@ -1018,11 +1115,12 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { } // return -int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { - if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) { +int32_t tmqHandleRes(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) { + if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) { /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ - if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) { - tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp); + if (rspHead->epoch > atomic_load_32(&tmq->epoch)) { + SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead; + tmqUpdateEp(tmq, rspHead->epoch, rspMsg); tmqClearUnhandleMsg(tmq); *pReset = true; } else { @@ -1036,21 +1134,22 @@ int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { while (1) { - tmq_message_t* rspMsg = NULL; - taosGetQitem(tmq->qall, (void**)&rspMsg); - if (rspMsg == NULL) { + SMqRspHead* rspHead = NULL; + taosGetQitem(tmq->qall, (void**)&rspHead); + if (rspHead == NULL) { taosReadAllQitems(tmq->mqueue, tmq->qall); - taosGetQitem(tmq->qall, (void**)&rspMsg); - if (rspMsg == NULL) return NULL; + taosGetQitem(tmq->qall, (void**)&rspHead); + if (rspHead == NULL) return NULL; } - if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { + if (rspHead->mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { + tmq_message_t* rspMsg = (tmq_message_t*)rspHead; atomic_sub_fetch_32(&tmq->readyRequest, 1); /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ - if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) { + if (rspMsg->msg.head.epoch == atomic_load_32(&tmq->epoch)) { /*printf("epoch match\n");*/ - SMqClientVg* pVg = rspMsg->extra; - pVg->currentOffset = rspMsg->consumeRsp.rspOffset; + SMqClientVg* pVg = rspMsg->vg; + pVg->currentOffset = rspMsg->msg.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); return rspMsg; } else { @@ -1060,8 +1159,8 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese } else { /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/ bool reset = false; - tmqHandleRes(tmq, rspMsg, &reset); - taosFreeQitem(rspMsg); + tmqHandleRes(tmq, rspHead, &reset); + taosFreeQitem(rspHead); if (pollIfReset && reset) { printf("reset and repoll\n"); tmqPollImpl(tmq, blockingTime); @@ -1070,6 +1169,7 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese } } +#if 0 tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* rspMsg = NULL; int64_t startTime = taosGetTimestampMs(); @@ -1092,6 +1192,7 @@ tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { return NULL; } } +#endif tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* rspMsg; @@ -1157,7 +1258,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { return NULL; } - SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* param = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (param == NULL) { ASSERT(false); taosMsleep(blocking_time); @@ -1188,7 +1289,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tsem_wait(¶m->rspSem); tsem_destroy(¶m->rspSem); - free(param); + taosMemoryFree(param); if (tmq_message == NULL) { if (beginVgIdx == pTopic->nextVgIdx) { @@ -1230,7 +1331,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); if (pParam == NULL) { continue; } @@ -1257,9 +1358,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - SMqPollRsp* pRsp = &tmq_message->consumeRsp; + SMqPollRsp* pRsp = &tmq_message->msg; tDeleteSMqConsumeRsp(pRsp); - /*free(tmq_message);*/ + /*taosMemoryFree(tmq_message);*/ taosFreeQitem(tmq_message); } @@ -1273,24 +1374,24 @@ const char* tmq_err2str(tmq_resp_err_t err) { } TAOS_ROW tmq_get_row(tmq_message_t* message) { - SMqPollRsp* rsp = &message->consumeRsp; + SMqPollRsp* rsp = &message->msg; while (1) { - if (message->curBlock < taosArrayGetSize(rsp->pBlockData)) { - SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->curBlock); - if (message->curRow < pBlock->info.rows) { + if (message->iter.curBlock < taosArrayGetSize(rsp->pBlockData)) { + SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->iter.curBlock); + if (message->iter.curRow < pBlock->info.rows) { for (int i = 0; i < pBlock->info.numOfCols; i++) { SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i); - if (colDataIsNull_s(pData, message->curRow)) - message->uData[i] = NULL; + if (colDataIsNull_s(pData, message->iter.curRow)) + message->iter.uData[i] = NULL; else { - message->uData[i] = colDataGetData(pData, message->curRow); + message->iter.uData[i] = colDataGetData(pData, message->iter.curRow); } } - message->curRow++; - return message->uData; + message->iter.curRow++; + return message->iter.uData; } else { - message->curBlock++; - message->curRow = 0; + message->iter.curBlock++; + message->iter.curRow = 0; continue; } } @@ -1302,7 +1403,7 @@ char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; #if 0 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { - tmq_t* pTmq = malloc(sizeof(tmq_t)); + tmq_t* pTmq = taosMemoryMalloc(sizeof(tmq_t)); if (pTmq == NULL) { return NULL; } @@ -1316,7 +1417,7 @@ tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } #endif diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 1a155beb90c9eb1d14eebb0d465b837ba35501b2..629dd90da3b531f373ff353ea6bd4123215f65aa 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -52,7 +52,7 @@ TEST(testCase, driverInit_Test) { // taosInitGlobalCfg(); // taos_init(); } - +#if 0 TEST(testCase, connect_Test) { // taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg"); @@ -271,6 +271,8 @@ TEST(testCase, create_stable_Test) { } taos_free_result(pRes); + pRes = taos_query(pConn, "use abc1"); + pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); if (taos_errno(pRes) != 0) { printf("error in create stable, reason:%s\n", taos_errstr(pRes)); @@ -283,17 +285,17 @@ TEST(testCase, create_stable_Test) { ASSERT_EQ(numOfFields, 0); taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); - } - - pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - pRes = taos_query(pConn, "drop stable `123_$^)`"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); - } +// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); +// } +// +// pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// pRes = taos_query(pConn, "drop stable `123_$^)`"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); +// } taos_close(pConn); } @@ -333,7 +335,7 @@ TEST(testCase, create_ctable_Test) { } taos_free_result(pRes); - pRes = taos_query(pConn, "create table tu using sts tags('2021-10-10 1:1:1');"); + pRes = taos_query(pConn, "create table tu using st1 tags('2021-10-10 1:1:1');"); if (taos_errno(pRes) != 0) { printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); } @@ -483,7 +485,9 @@ TEST(testCase, show_table_Test) { taos_free_result(pRes); - pRes = taos_query(pConn, "show abc1.tables"); + taos_query(pConn, "use abc1"); + + pRes = taos_query(pConn, "show tables"); if (taos_errno(pRes) != 0) { printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -648,6 +652,7 @@ TEST(testCase, projection_query_stables) { taos_free_result(pRes); taos_close(pConn); } +#endif TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -656,13 +661,7 @@ TEST(testCase, agg_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "create table tx using st1 tags(111111111111111)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "select count(*) from t_x_19"); + pRes = taos_query(pConn, "select count(*) from tu"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 31328cc4b2b6dea97bf7661ce3aeb7e80fa3ca1f..c624e383fb5210a51c7f882b547f783841476fd7 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -18,14 +18,14 @@ #include "tcompare.h" #include "tglobal.h" -int32_t taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { +int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; strcpy(pEp->fqdn, ep); - char *temp = strchr(pEp->fqdn, ':'); + char* temp = strchr(pEp->fqdn, ':'); if (temp) { *temp = 0; - pEp->port = atoi(temp+1); + pEp->port = atoi(temp + 1); } if (pEp->port == 0) { @@ -36,7 +36,7 @@ int32_t taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { return 0; } -void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) { +void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) { if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { return; } @@ -47,26 +47,25 @@ void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) { pEpSet->numOfEps += 1; } -bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) { +bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; } for (int32_t i = 0; i < s1->numOfEps; i++) { - if (s1->eps[i].port != s2->eps[i].port - || strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0) + if (s1->eps[i].port != s2->eps[i].port || strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0) return false; } return true; } -void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet) { +void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet) { taosCorBeginWrite(&pEpSet->version); pEpSet->epSet = *pNewEpSet; taosCorEndWrite(&pEpSet->version); } -SEpSet getEpSet_s(SCorEpSet *pEpSet) { +SEpSet getEpSet_s(SCorEpSet* pEpSet) { SEpSet ep = {0}; taosCorBeginRead(&pEpSet->version); ep = pEpSet->epSet; @@ -75,7 +74,6 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) { return ep; } - int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -95,7 +93,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con if (isNull) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. + pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. } else { colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); } @@ -113,11 +111,11 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con newSize = 8; } - while(newSize < pAttr->length + varDataTLen(pData)) { + while (newSize < pAttr->length + varDataTLen(pData)) { newSize = newSize * 1.5; } - char* buf = realloc(pColumnInfoData->pData, newSize); + char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (buf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -133,19 +131,40 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con pColumnInfoData->varmeta.length += varDataTLen(pData); } else { char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; - switch(type) { - case TSDB_DATA_TYPE_BOOL: {*(bool*) p = *(bool*) pData;break;} + switch (type) { + case TSDB_DATA_TYPE_BOOL: { + *(bool*)p = *(bool*)pData; + break; + } case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: {*(int8_t*) p = *(int8_t*) pData;break;} + case TSDB_DATA_TYPE_UTINYINT: { + *(int8_t*)p = *(int8_t*)pData; + break; + } case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: {*(int16_t*) p = *(int16_t*) pData;break;} + case TSDB_DATA_TYPE_USMALLINT: { + *(int16_t*)p = *(int16_t*)pData; + break; + } case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: {*(int32_t*) p = *(int32_t*) pData;break;} + case TSDB_DATA_TYPE_UINT: { + *(int32_t*)p = *(int32_t*)pData; + break; + } case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: {*(int64_t*) p = *(int64_t*) pData;break;} - case TSDB_DATA_TYPE_FLOAT: {*(float*) p = *(float*) pData;break;} - case TSDB_DATA_TYPE_DOUBLE: {*(double*) p = *(double*) pData;break;} + case TSDB_DATA_TYPE_UBIGINT: { + *(int64_t*)p = *(int64_t*)pData; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + *(float*)p = *(float*)pData; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + *(double*)p = *(double*)pData; + break; + } default: assert(0); } @@ -154,11 +173,12 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con return 0; } -static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, int32_t numOfRow2) { +static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, + int32_t numOfRow2) { uint32_t total = numOfRow1 + numOfRow2; if (BitmapLen(numOfRow1) < BitmapLen(total)) { - char* tmp = realloc(pColumnInfoData->nullbitmap, BitmapLen(total)); + char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(total)); uint32_t extend = BitmapLen(total) - BitmapLen(numOfRow1); memset(tmp + BitmapLen(numOfRow1), 0, extend); pColumnInfoData->nullbitmap = tmp; @@ -188,7 +208,8 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2) { +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, + uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRow2 == 0) { @@ -197,13 +218,13 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - char* p = realloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); if (p == NULL) { // TODO } - pColumnInfoData->varmeta.offset = (int32_t*) p; - for(int32_t i = 0; i < numOfRow2; ++i) { + pColumnInfoData->varmeta.offset = (int32_t*)p; + for (int32_t i = 0; i < numOfRow2; ++i) { pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length; } @@ -211,7 +232,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co uint32_t len = pSource->varmeta.length; uint32_t oldLen = pColumnInfoData->varmeta.length; if (pColumnInfoData->varmeta.allocLen < len + oldLen) { - char* tmp = realloc(pColumnInfoData->pData, len + oldLen); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } @@ -226,7 +247,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); int32_t newSize = (numOfRow1 + numOfRow2) * pColumnInfoData->info.bytes; - char* tmp = realloc(pColumnInfoData->pData, newSize); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } @@ -239,14 +260,62 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co return numOfRow1 + numOfRow2; } +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) { + ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); + if (numOfRows == 0) { + return numOfRows; + } + + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + // Handle the bitmap + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumnInfoData->varmeta.offset = (int32_t*)p; + memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); + + if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumnInfoData->pData = tmp; + pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; + } + + memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); + pColumnInfoData->varmeta.length = pSource->varmeta.length; + } else { + char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumnInfoData->nullbitmap = tmp; + memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); + + int32_t newSize = numOfRows * pColumnInfoData->info.bytes; + tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumnInfoData->pData = tmp; + memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); + } + + return 0; +} + size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock)); return pBlock->info.numOfCols; } -size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { - return pBlock->info.rows; -} +size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { if (pDataBlock == NULL || pDataBlock->info.rows <= 0) { @@ -263,8 +332,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { } ASSERT(pColInfoData->nullbitmap == NULL); - pDataBlock->info.window.skey = *(TSKEY*) colDataGetData(pColInfoData, 0); - pDataBlock->info.window.ekey = *(TSKEY*) colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); + pDataBlock->info.window.skey = *(TSKEY*)colDataGetData(pColInfoData, 0); + pDataBlock->info.window.ekey = *(TSKEY*)colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); return 0; } @@ -272,7 +341,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); int32_t numOfCols = pSrc->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); @@ -280,7 +349,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); int32_t newSize = oldLen + newLen; - char* tmp = realloc(pCol2->pData, newSize); + char* tmp = taosMemoryRealloc(pCol2->pData, newSize); if (tmp != NULL) { pCol2->pData = tmp; colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); @@ -296,10 +365,10 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { size_t blockDataGetSize(const SSDataBlock* pBlock) { assert(pBlock != NULL); - size_t total = 0; + size_t total = 0; int32_t numOfCols = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); total += colDataGetLength(pColInfoData, pBlock->info.rows); @@ -315,7 +384,8 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) { // the number of tuples can be fit in one page. // Actual data rows pluses the corresponding meta data must fit in one memory buffer of the given page size. -int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize) { +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, + int32_t pageSize) { ASSERT(pBlock != NULL && stopIndex != NULL); int32_t numOfCols = pBlock->info.numOfCols; @@ -323,13 +393,13 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd int32_t bitmapChar = 1; - size_t headerSize = sizeof(int32_t); + size_t headerSize = sizeof(int32_t); size_t colHeaderSize = sizeof(int32_t) * numOfCols; - size_t payloadSize = pageSize - (headerSize + colHeaderSize); + size_t payloadSize = pageSize - (headerSize + colHeaderSize); // TODO speedup by checking if the whole page can fit in firstly. if (!hasVarCol) { - size_t rowSize = blockDataGetRowSize(pBlock); + size_t rowSize = blockDataGetRowSize(pBlock); int32_t capacity = (payloadSize / (rowSize * 8 + bitmapChar * numOfCols)) * 8; *stopIndex = startIndex + capacity; @@ -342,7 +412,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd // iterate the rows that can be fit in this buffer page int32_t size = (headerSize + colHeaderSize); - for(int32_t j = startIndex; j < numOfRows; ++j) { + for (int32_t j = startIndex; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { @@ -359,7 +429,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd size += pColInfoData->info.bytes; if (((j - startIndex) & 0x07) == 0) { - size += 1; // the space for null bitmap + size += 1; // the space for null bitmap } } } @@ -383,7 +453,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return NULL; } - SSDataBlock* pDst = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pDst == NULL) { return NULL; } @@ -393,17 +463,17 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 pDst->info.rows = 0; pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); colInfo.info = pSrcCol->info; if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) { SVarColAttr* pAttr = &colInfo.varmeta; - pAttr->offset = calloc(rowCount, sizeof(int32_t)); + pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t)); } else { - colInfo.nullbitmap = calloc(1, BitmapLen(rowCount)); - colInfo.pData = calloc(rowCount, colInfo.info.bytes); + colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount)); + colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes); } taosArrayPush(pDst->pDataBlock, &colInfo); @@ -414,7 +484,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { - bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); + bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); char* p = colDataGetData(pColData, j); colDataAppend(pDstCol, j - startIndex, p, isNull); @@ -427,11 +497,11 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 /** * - * +------------------+---------------+--------------------+ - * |the number of rows| column length | column #1 | - * | (4 bytes) | (4 bytes) |--------------------+ - * | | | null bitmap| values| - * +------------------+---------------+--------------------+ + * +------------------+---------------------------------------------+ + * |the number of rows| column #1 | + * | (4 bytes) |------------+-----------------------+--------+ + * | | null bitmap| column length(4bytes) | values | + * +------------------+------------+-----------------------+--------+ * @param buf * @param pBlock * @return @@ -440,14 +510,14 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); // write the number of rows - *(uint32_t*) buf = pBlock->info.rows; + *(uint32_t*)buf = pBlock->info.rows; int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfRows = pBlock->info.rows; char* pStart = buf + sizeof(uint32_t); - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pCol->info.type)) { memcpy(pStart, pCol->varmeta.offset, numOfRows * sizeof(int32_t)); @@ -459,7 +529,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { uint32_t dataSize = colDataGetLength(pCol, numOfRows); - *(int32_t*) pStart = dataSize; + *(int32_t*)pStart = dataSize; pStart += sizeof(int32_t); memcpy(pStart, pCol->pData, dataSize); @@ -470,12 +540,12 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { } int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { - pBlock->info.rows = *(int32_t*) buf; + pBlock->info.rows = *(int32_t*)buf; - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = pBlock->info.numOfCols; const char* pStart = buf + sizeof(uint32_t); - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); size_t metaSize = pBlock->info.rows * sizeof(int32_t); @@ -487,12 +557,12 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { pStart += BitmapLen(pBlock->info.rows); } - int32_t colLength = *(int32_t*) pStart; + int32_t colLength = *(int32_t*)pStart; pStart += sizeof(int32_t); if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (pCol->varmeta.allocLen < colLength) { - char* tmp = realloc(pCol->pData, colLength); + char* tmp = taosMemoryRealloc(pCol->pData, colLength); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -512,17 +582,21 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { return TSDB_CODE_SUCCESS; } -size_t blockDataGetRowSize(const SSDataBlock* pBlock) { +size_t blockDataGetRowSize(SSDataBlock* pBlock) { ASSERT(pBlock != NULL); - size_t rowSize = 0; + if (pBlock->info.rowSize == 0) { + size_t rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - rowSize += pColInfo->info.bytes; + size_t numOfCols = pBlock->info.numOfCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + rowSize += pColInfo->info.bytes; + } + + pBlock->info.rowSize = rowSize; } - return rowSize; + return pBlock->info.rowSize; } /** @@ -536,11 +610,11 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { } SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) { - SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema)); - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SSchema* pSchema = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(SSchema)); + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pSchema[i].bytes = pColInfoData->info.bytes; - pSchema[i].type = pColInfoData->info.type; + pSchema[i].type = pColInfoData->info.type; pSchema[i].colId = pColInfoData->info.colId; } @@ -556,14 +630,14 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { double rowSize = 0; size_t numOfCols = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { rowSize += sizeof(int32_t); } else { - rowSize += 1/8.0; + rowSize += 1 / 8.0; // one bit for each record } } @@ -571,56 +645,56 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { } typedef struct SSDataBlockSortHelper { - SArray *orderInfo; // SArray - SSDataBlock *pDataBlock; + SArray* orderInfo; // SArray + SSDataBlock* pDataBlock; bool nullFirst; } SSDataBlockSortHelper; int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { - const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*) param; + const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param; SSDataBlock* pDataBlock = pHelper->pDataBlock; - int32_t left = *(int32_t*) p1; - int32_t right = *(int32_t*) p2; + int32_t left = *(int32_t*)p1; + int32_t right = *(int32_t*)p2; SArray* pInfo = pHelper->orderInfo; - for(int32_t i = 0; i < pInfo->size; ++i) { + for (int32_t i = 0; i < pInfo->size; ++i) { SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); - SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); + SColumnInfoData* pColInfoData = pOrder->pColData; // TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); if (pColInfoData->hasNull) { - bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); + bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); if (leftNull && rightNull) { - continue; // continue to next slot + continue; // continue to next slot } if (rightNull) { - return pHelper->nullFirst? 1:-1; + return pHelper->nullFirst ? 1 : -1; } if (leftNull) { - return pHelper->nullFirst? -1:1; + return pHelper->nullFirst ? -1 : 1; } } - void* left1 = colDataGetData(pColInfoData, left); + void* left1 = colDataGetData(pColInfoData, left); void* right1 = colDataGetData(pColInfoData, right); - switch(pColInfoData->info.type) { + switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_INT: { - int32_t leftx = *(int32_t*) left1; - int32_t rightx = *(int32_t*) right1; + int32_t leftx = *(int32_t*)left1; + int32_t rightx = *(int32_t*)right1; if (leftx == rightx) { break; } else { if (pOrder->order == TSDB_ORDER_ASC) { - return (leftx < rightx)? -1:1; + return (leftx < rightx) ? -1 : 1; } else { - return (leftx < rightx)? 1:-1; + return (leftx < rightx) ? 1 : -1; } } } @@ -632,7 +706,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { return 0; } -static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { +static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, + int32_t tupleIndex) { int32_t code = 0; int32_t numOfCols = pSrcBlock->info.numOfCols; @@ -666,17 +741,17 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } } #else - for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pSrc->info.type)) { - memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); - pDst->varmeta.length = pSrc->varmeta.length; + memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); + pDst->varmeta.length = pSrc->varmeta.length; - for(int32_t j = 0; j < pDataBlock->info.rows; ++j) { - pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]]; - } + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]]; + } } else { switch (pSrc->info.type) { case TSDB_DATA_TYPE_UINT: @@ -744,24 +819,24 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { int32_t rows = pDataBlock->info.rows; int32_t numOfCols = pDataBlock->info.numOfCols; - SColumnInfoData* pCols = calloc(numOfCols, sizeof(SColumnInfoData)); + SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { return NULL; } - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); pCols[i].info = pColInfoData->info; if (IS_VAR_DATA_TYPE(pCols[i].info.type)) { - pCols[i].varmeta.offset = calloc(rows, sizeof(int32_t)); - pCols[i].pData = calloc(1, pColInfoData->varmeta.length); + pCols[i].varmeta.offset = taosMemoryCalloc(rows, sizeof(int32_t)); + pCols[i].pData = taosMemoryCalloc(1, pColInfoData->varmeta.length); pCols[i].varmeta.length = pColInfoData->varmeta.length; pCols[i].varmeta.allocLen = pCols[i].varmeta.length; } else { - pCols[i].nullbitmap = calloc(1, BitmapLen(rows)); - pCols[i].pData = calloc(rows, pCols[i].info.bytes); + pCols[i].nullbitmap = taosMemoryCalloc(1, BitmapLen(rows)); + pCols[i].pData = taosMemoryCalloc(rows, pCols[i].info.bytes); } } @@ -771,56 +846,64 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { int32_t numOfCols = pDataBlock->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); pColInfoData->info = pCols[i].info; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - tfree(pColInfoData->varmeta.offset); + taosMemoryFreeClear(pColInfoData->varmeta.offset); pColInfoData->varmeta = pCols[i].varmeta; } else { - tfree(pColInfoData->nullbitmap); + taosMemoryFreeClear(pColInfoData->nullbitmap); pColInfoData->nullbitmap = pCols[i].nullbitmap; } - tfree(pColInfoData->pData); + taosMemoryFreeClear(pColInfoData->pData); pColInfoData->pData = pCols[i].pData; } - tfree(pCols); + taosMemoryFreeClear(pCols); } static int32_t* createTupleIndex(size_t rows) { - int32_t* index = calloc(rows, sizeof(int32_t)); + int32_t* index = taosMemoryCalloc(rows, sizeof(int32_t)); if (index == NULL) { return NULL; } - for(int32_t i = 0; i < rows; ++i) { + for (int32_t i = 0; i < rows; ++i) { index[i] = i; } return index; } -static void destroyTupleIndex(int32_t* index) { - tfree(index); -} +static void destroyTupleIndex(int32_t* index) { taosMemoryFreeClear(index); } static __compar_fn_t getComparFn(int32_t type, int32_t order) { - switch(type) { - case TSDB_DATA_TYPE_TINYINT: return order == TSDB_ORDER_ASC? compareInt8Val:compareInt8ValDesc; - case TSDB_DATA_TYPE_SMALLINT: return order == TSDB_ORDER_ASC? compareInt16Val:compareInt16ValDesc; - case TSDB_DATA_TYPE_INT: return order == TSDB_ORDER_ASC? compareInt32Val:compareInt32ValDesc; - case TSDB_DATA_TYPE_BIGINT: return order == TSDB_ORDER_ASC? compareInt64Val:compareInt64ValDesc; - case TSDB_DATA_TYPE_FLOAT: return order == TSDB_ORDER_ASC? compareFloatVal:compareFloatValDesc; - case TSDB_DATA_TYPE_DOUBLE: return order == TSDB_ORDER_ASC? compareDoubleVal:compareDoubleValDesc; - case TSDB_DATA_TYPE_UTINYINT: return order == TSDB_ORDER_ASC? compareUint8Val:compareUint8ValDesc; - case TSDB_DATA_TYPE_USMALLINT:return order == TSDB_ORDER_ASC? compareUint16Val:compareUint16ValDesc; - case TSDB_DATA_TYPE_UINT: return order == TSDB_ORDER_ASC? compareUint32Val:compareUint32ValDesc; - case TSDB_DATA_TYPE_UBIGINT: return order == TSDB_ORDER_ASC? compareUint64Val:compareUint64ValDesc; + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + return order == TSDB_ORDER_ASC ? compareInt8Val : compareInt8ValDesc; + case TSDB_DATA_TYPE_SMALLINT: + return order == TSDB_ORDER_ASC ? compareInt16Val : compareInt16ValDesc; + case TSDB_DATA_TYPE_INT: + return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc; + case TSDB_DATA_TYPE_BIGINT: + return order == TSDB_ORDER_ASC ? compareInt64Val : compareInt64ValDesc; + case TSDB_DATA_TYPE_FLOAT: + return order == TSDB_ORDER_ASC ? compareFloatVal : compareFloatValDesc; + case TSDB_DATA_TYPE_DOUBLE: + return order == TSDB_ORDER_ASC ? compareDoubleVal : compareDoubleValDesc; + case TSDB_DATA_TYPE_UTINYINT: + return order == TSDB_ORDER_ASC ? compareUint8Val : compareUint8ValDesc; + case TSDB_DATA_TYPE_USMALLINT: + return order == TSDB_ORDER_ASC ? compareUint16Val : compareUint16ValDesc; + case TSDB_DATA_TYPE_UINT: + return order == TSDB_ORDER_ASC ? compareUint32Val : compareUint32ValDesc; + case TSDB_DATA_TYPE_UBIGINT: + return order == TSDB_ORDER_ASC ? compareUint64Val : compareUint64ValDesc; default: - return order == TSDB_ORDER_ASC? compareInt32Val:compareInt32ValDesc; + return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc; } } @@ -865,10 +948,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs return TSDB_CODE_SUCCESS; } else { // var data type - } } else if (pDataBlock->info.numOfCols == 2) { - } } @@ -881,7 +962,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs int64_t p0 = taosGetTimestampUs(); SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; - for(int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i); pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex); } @@ -909,7 +990,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1-p0, p2 - p1, p3 - p2, p4-p3, rows); + printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, + p3 - p2, p4 - p3, rows); destroyTupleIndex(index); return TSDB_CODE_SUCCESS; @@ -917,14 +999,18 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs typedef struct SHelper { int32_t index; - union {char *pData; int64_t i64; double d64;}; + union { + char* pData; + int64_t i64; + double d64; + }; } SHelper; SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* pBlock) { int32_t sortValLengthPerRow = 0; int32_t numOfCols = taosArrayGetSize(pOrderInfo); - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); pInfo->pColData = pColInfo; @@ -933,19 +1019,20 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* size_t len = sortValLengthPerRow * pBlock->info.rows; - char* buf = calloc(1, len); - SHelper* phelper = calloc(numOfRows, sizeof(SHelper)); - for(int32_t i = 0; i < numOfRows; ++i) { + char* buf = taosMemoryCalloc(1, len); + SHelper* phelper = taosMemoryCalloc(numOfRows, sizeof(SHelper)); + for (int32_t i = 0; i < numOfRows; ++i) { phelper[i].index = i; phelper[i].pData = buf + sortValLengthPerRow * i; } int32_t offset = 0; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); - for(int32_t j = 0; j < numOfRows; ++j) { - phelper[j].i64 = *(int32_t*) pInfo->pColData->pData + pInfo->pColData->info.bytes * j; -// memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j, pInfo->pColData->info.bytes); + for (int32_t j = 0; j < numOfRows; ++j) { + phelper[j].i64 = *(int32_t*)pInfo->pColData->pData + pInfo->pColData->info.bytes * j; + // memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j, + // pInfo->pColData->info.bytes); } offset += pInfo->pColData->info.bytes; @@ -955,70 +1042,68 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* } int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { - const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*) param; + const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param; -// SSDataBlock* pDataBlock = pHelper->pDataBlock; + // SSDataBlock* pDataBlock = pHelper->pDataBlock; - SHelper* left = (SHelper*) p1; - SHelper* right = (SHelper*) p2; + SHelper* left = (SHelper*)p1; + SHelper* right = (SHelper*)p2; SArray* pInfo = pHelper->orderInfo; int32_t offset = 0; -// for(int32_t i = 0; i < pInfo->size; ++i) { -// SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0); -// SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); - -// if (pColInfoData->hasNull) { -// bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); -// bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); -// if (leftNull && rightNull) { -// continue; // continue to next slot -// } -// -// if (rightNull) { -// return pHelper->nullFirst? 1:-1; -// } -// -// if (leftNull) { -// return pHelper->nullFirst? -1:1; -// } -// } - -// void* left1 = colDataGetData(pColInfoData, left); -// void* right1 = colDataGetData(pColInfoData, right); - -// switch(pColInfoData->info.type) { -// case TSDB_DATA_TYPE_INT: { - int32_t leftx = *(int32_t*)left->pData;//*(int32_t*)(left->pData + offset); - int32_t rightx = *(int32_t*)right->pData;//*(int32_t*)(right->pData + offset); - -// offset += pColInfoData->info.bytes; - if (leftx == rightx) { -// break; - return 0; - } else { -// if (pOrder->order == TSDB_ORDER_ASC) { - return (leftx < rightx)? -1:1; -// } else { -// return (leftx < rightx)? 1:-1; -// } - } -// } -// default: -// assert(0); -// } -// } + // for(int32_t i = 0; i < pInfo->size; ++i) { + // SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0); + // SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); + + // if (pColInfoData->hasNull) { + // bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); + // bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); + // if (leftNull && rightNull) { + // continue; // continue to next slot + // } + // + // if (rightNull) { + // return pHelper->nullFirst? 1:-1; + // } + // + // if (leftNull) { + // return pHelper->nullFirst? -1:1; + // } + // } + + // void* left1 = colDataGetData(pColInfoData, left); + // void* right1 = colDataGetData(pColInfoData, right); + + // switch(pColInfoData->info.type) { + // case TSDB_DATA_TYPE_INT: { + int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset); + int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset); + + // offset += pColInfoData->info.bytes; + if (leftx == rightx) { + // break; + return 0; + } else { + // if (pOrder->order == TSDB_ORDER_ASC) { + return (leftx < rightx) ? -1 : 1; + // } else { + // return (leftx < rightx)? 1:-1; + // } + } + // } + // default: + // assert(0); + // } + // } return 0; } -int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { - return 0; -} +int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { return 0; } int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { -// Allocate the additional buffer. + // Allocate the additional buffer. int64_t p0 = taosGetTimestampUs(); SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; @@ -1032,7 +1117,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF taosqsort(index, rows, sizeof(SHelper), &helper, dataBlockCompar_rv); - int64_t p1 = taosGetTimestampUs(); + int64_t p1 = taosGetTimestampUs(); SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); if (pCols == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1052,12 +1137,13 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - printf("sort:%" PRId64 ", create:%" PRId64", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); + printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, + p3 - p2, p4 - p3, rows); // destroyTupleIndex(index); return 0; } -void blockDataClearup(SSDataBlock* pDataBlock) { +void blockDataCleanup(SSDataBlock* pDataBlock) { pDataBlock->info.rows = 0; if (pDataBlock->info.hasVarCol) { @@ -1077,7 +1163,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo } if (IS_VAR_DATA_TYPE(pColumn->info.type)) { - char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); + char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1087,9 +1173,9 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo pColumn->varmeta.length = 0; pColumn->varmeta.allocLen = 0; - tfree(pColumn->pData); + taosMemoryFreeClear(pColumn->pData); } else { - char* tmp = realloc(pColumn->nullbitmap, BitmapLen(numOfRows)); + char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1097,7 +1183,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo pColumn->nullbitmap = tmp; memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); assert(pColumn->info.bytes); - tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); + tmp = taosMemoryRealloc(pColumn->pData, numOfRows * pColumn->info.bytes); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1110,8 +1196,8 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; - - for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); code = blockDataEnsureColumnCapacity(p, numOfRows); if (code) { @@ -1127,34 +1213,21 @@ void* blockDataDestroy(SSDataBlock* pBlock) { return NULL; } - int32_t numOfOutput = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - tfree(pColInfoData->varmeta.offset); - } else { - tfree(pColInfoData->nullbitmap); - } - - tfree(pColInfoData->pData); - } - - taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockAgg); - // tfree(pBlock); + blockDestroyInner(pBlock); + taosMemoryFreeClear(pBlock); return NULL; } SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { int32_t numOfCols = pDataBlock->info.numOfCols; - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.numOfCols = numOfCols; pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); colInfo.info = p->info; @@ -1230,4 +1303,116 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { taosArrayPush(pBlock->pDataBlock, &data); } return (void*)buf; -} \ No newline at end of file +} + +int32_t tEncodeDataBlocks(void** buf, const SArray* blocks) { + int32_t tlen = 0; + int32_t sz = taosArrayGetSize(blocks); + tlen += taosEncodeFixedI32(buf, sz); + + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pBlock = taosArrayGet(blocks, i); + tlen += tEncodeDataBlock(buf, pBlock); + } + + return tlen; +} + +void* tDecodeDataBlocks(const void* buf, SArray** blocks) { + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + + *blocks = taosArrayInit(sz, sizeof(SSDataBlock)); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock pBlock = {0}; + buf = tDecodeDataBlock(buf, &pBlock); + taosArrayPush(*blocks, &pBlock); + } + return (void*)buf; +} + +static char* formatTimestamp(char* buf, int64_t val, int precision) { + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm* ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} +void blockDebugShowData(const SArray* dataBlocks) { + char pBuf[128]; + int32_t sz = taosArrayGetSize(dataBlocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + for (int32_t j = 0; j < rows; j++) { + printf("|"); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + printf(" %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + printf(" %15d |", *(int32_t*)var); + break; + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t*)var); + break; + case TSDB_DATA_TYPE_BIGINT: + printf(" %15ld |", *(int64_t*)var); + break; + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15lu |", *(uint64_t*)var); + break; + } + } + printf("\n"); + } + } +} + diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f39b20b9347e7f1a441cd51d01c5297ebafef024..5b27fd01f4718206fa720e9fdfcef47d4ce02420 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -37,7 +37,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { #endif if (pCol->spaceSize < spaceNeeded) { - void *ptr = realloc(pCol->pData, spaceNeeded); + void *ptr = taosMemoryRealloc(pCol->pData, spaceNeeded); if (ptr == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, strerror(errno)); return -1; @@ -66,7 +66,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { */ STSchema *tdDupSchema(const STSchema *pSchema) { int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); - STSchema *tSchema = (STSchema *)malloc(tlen); + STSchema *tSchema = (STSchema *)taosMemoryMalloc(tlen); if (tSchema == NULL) return NULL; memcpy((void *)tSchema, (void *)pSchema, tlen); @@ -127,7 +127,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { if (pBuilder == NULL) return -1; pBuilder->tCols = 256; - pBuilder->columns = (STColumn *)malloc(sizeof(STColumn) * pBuilder->tCols); + pBuilder->columns = (STColumn *)taosMemoryMalloc(sizeof(STColumn) * pBuilder->tCols); if (pBuilder->columns == NULL) return -1; tdResetTSchemaBuilder(pBuilder, version); @@ -136,7 +136,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { if (pBuilder) { - tfree(pBuilder->columns); + taosMemoryFreeClear(pBuilder->columns); } } @@ -153,7 +153,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int1 if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; - STColumn *columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); + STColumn *columns = (STColumn *)taosMemoryRealloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); if (columns == NULL) return -1; pBuilder->columns = columns; } @@ -191,7 +191,7 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { int tlen = sizeof(STSchema) + sizeof(STColumn) * pBuilder->nCols; - STSchema *pSchema = (STSchema *)malloc(tlen); + STSchema *pSchema = (STSchema *)taosMemoryMalloc(tlen); if (pSchema == NULL) return NULL; schemaVersion(pSchema) = pBuilder->version; @@ -221,7 +221,7 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema) { SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { int32_t size = dataRowMaxBytesFromSchema(pSchema); - SDataRow row = malloc(size); + SDataRow row = taosMemoryMalloc(size); if (row == NULL) return NULL; tdInitDataRow(row, pSchema); @@ -232,11 +232,11 @@ SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { * Free the SDataRow object */ void tdFreeDataRow(SDataRow row) { - if (row) free(row); + if (row) taosMemoryFree(row); } SDataRow tdDataRowDup(SDataRow row) { - SDataRow trow = malloc(dataRowLen(row)); + SDataRow trow = taosMemoryMalloc(dataRowLen(row)); if (trow == NULL) return NULL; dataRowCpy(trow, row); @@ -244,7 +244,7 @@ SDataRow tdDataRowDup(SDataRow row) { } SMemRow tdMemRowDup(SMemRow row) { - SMemRow trow = malloc(memRowTLen(row)); + SMemRow trow = taosMemoryMalloc(memRowTLen(row)); if (trow == NULL) return NULL; memRowCpy(trow, row); @@ -348,7 +348,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) { } SDataCols *tdNewDataCols(int maxCols, int maxRows) { - SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols)); + SDataCols *pCols = (SDataCols *)taosMemoryCalloc(1, sizeof(SDataCols)); if (pCols == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno)); return NULL; @@ -360,7 +360,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->numOfCols = 0; if (maxCols > 0) { - pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); + pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol)); if (pCols->cols == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); @@ -384,7 +384,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { int oldMaxCols = pCols->maxCols; if (schemaNCols(pSchema) > oldMaxCols) { pCols->maxCols = schemaNCols(pSchema); - void *ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; for (i = oldMaxCols; i < pCols->maxCols; i++) { @@ -411,16 +411,17 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { int maxCols = pCols->maxCols; for (i = 0; i < maxCols; i++) { SDataCol *pCol = &pCols->cols[i]; - tfree(pCol->pData); + taosMemoryFreeClear(pCol->pData); } - free(pCols->cols); + taosMemoryFree(pCols->cols); pCols->cols = NULL; } - free(pCols); + taosMemoryFree(pCols); } return NULL; } +#if 0 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); if (pRet == NULL) return NULL; @@ -431,6 +432,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { for (int i = 0; i < pDataCols->numOfCols; i++) { pRet->cols[i].type = pDataCols->cols[i].type; + pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; pRet->cols[i].colId = pDataCols->cols[i].colId; pRet->cols[i].bytes = pDataCols->cols[i].bytes; pRet->cols[i].offset = pDataCols->cols[i].offset; @@ -453,6 +455,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { return pRet; } +#endif void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { @@ -638,7 +641,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i #endif SKVRow tdKVRowDup(SKVRow row) { - SKVRow trow = malloc(kvRowLen(row)); + SKVRow trow = taosMemoryMalloc(kvRowLen(row)); if (trow == NULL) return NULL; kvRowCpy(trow, row); @@ -671,7 +674,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { int oRowCols = kvRowNCols(row); ASSERT(diff > 0); - nrow = malloc(nRowLen); + nrow = taosMemoryMalloc(nRowLen); if (nrow == NULL) return -1; kvRowSetLen(nrow, nRowLen); @@ -689,7 +692,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { tdSortKVRowByColIdx(nrow); *orow = nrow; - free(row); + taosMemoryFree(row); } else { ASSERT(((SColIdx *)ptr)->colId == colId); if (IS_VAR_DATA_TYPE(type)) { @@ -700,7 +703,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { } else { // need to reallocate the memory int16_t nlen = kvRowLen(row) + (varDataTLen(value) - varDataTLen(pOldVal)); ASSERT(nlen > 0); - nrow = malloc(nlen); + nrow = taosMemoryMalloc(nlen); if (nrow == NULL) return -1; kvRowSetLen(nrow, nlen); @@ -725,7 +728,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { } *orow = nrow; - free(row); + taosMemoryFree(row); } } else { memcpy(kvRowColVal(row, (SColIdx *)ptr), value, TYPE_BYTES[type]); @@ -754,21 +757,21 @@ void *tdDecodeKVRow(void *buf, SKVRow *row) { int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) { pBuilder->tCols = 128; pBuilder->nCols = 0; - pBuilder->pColIdx = (SColIdx *)malloc(sizeof(SColIdx) * pBuilder->tCols); + pBuilder->pColIdx = (SColIdx *)taosMemoryMalloc(sizeof(SColIdx) * pBuilder->tCols); if (pBuilder->pColIdx == NULL) return -1; pBuilder->alloc = 1024; pBuilder->size = 0; - pBuilder->buf = malloc(pBuilder->alloc); + pBuilder->buf = taosMemoryMalloc(pBuilder->alloc); if (pBuilder->buf == NULL) { - free(pBuilder->pColIdx); + taosMemoryFree(pBuilder->pColIdx); return -1; } return 0; } void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) { - tfree(pBuilder->pColIdx); - tfree(pBuilder->buf); + taosMemoryFreeClear(pBuilder->pColIdx); + taosMemoryFreeClear(pBuilder->buf); } void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { @@ -782,7 +785,7 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { tlen += TD_KV_ROW_HEAD_SIZE; - SKVRow row = malloc(tlen); + SKVRow row = taosMemoryMalloc(tlen); if (row == NULL) return NULL; kvRowSetNCols(row, pBuilder->nCols); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9a0309e0245a517b452c25a6fff028755f8f7aa9..a1bef49cc6661705b7f51d400a9d553d45c4a3c9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -45,6 +45,7 @@ float tsRatioOfQueryCores = 1.0f; int32_t tsMaxBinaryDisplayWidth = 30; bool tsEnableSlaveQuery = 1; bool tsPrintAuth = 0; +int32_t tsMultiProcess = 0; // monitor bool tsEnableMonitor = 1; @@ -121,7 +122,7 @@ bool tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name bool tsKeepOriginalColumnName = 0; -// long query death-lock +// kill long query bool tsDeadLockKillQuery = 0; // tsdb config @@ -131,6 +132,9 @@ bool tsdbForceKeepFile = false; int32_t tsDiskCfgNum = 0; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; +// stream scheduler +bool tsStreamSchedV = true; + /* * minimum scale for whole system, millisecond by default * for TSDB_TIME_PRECISION_MILLI: 86400000L @@ -176,6 +180,10 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) { memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); if (pCfg->level == 0 && pCfg->primary == 1) { tstrncpy(tsDataDir, pCfg->dir, PATH_MAX); + if (taosMkDir(tsDataDir) != 0) { + uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); + return -1; + } } if (taosMkDir(pCfg->dir) != 0) { uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); @@ -309,7 +317,6 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1; if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1; - if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; if (cfgAddString(pCfg, "version", version, 1) != 0) return -1; if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1; @@ -340,6 +347,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1; if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1; @@ -403,7 +411,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } - tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; + tsNumOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval; tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; @@ -457,6 +465,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval; + tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->i32; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; @@ -583,4 +592,4 @@ void taosCfgDynamicOptions(const char *option, const char *value) { taosResetLog(); cfgDumpCfg(tsCfg, 1, false); } -} \ No newline at end of file +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d9242cf0ac68b65c264d2c74028f9a513eb7f3c6..61d012ad9788e5cc3fb1ebf8c35b97d345dff174 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -28,7 +28,7 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) { +int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; @@ -287,10 +287,11 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->dbFName); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); - tlen += taosEncodeFixedU8(buf, pReq->type); + tlen += taosEncodeFixedU8(buf, pReq->info); switch (pReq->type) { case TD_SUPER_TABLE: @@ -309,6 +310,20 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); } + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols); + for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]); + } + if (pReq->rollup && pReq->stbCfg.pRSmaParam) { + SRSmaParam *param = pReq->stbCfg.pRSmaParam; + tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor); + tlen += taosEncodeFixedI8(buf, param->delayUnit); + tlen += taosEncodeFixedI8(buf, param->nFuncIds); + for (int8_t i = 0; i < param->nFuncIds; ++i) { + tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); + } + tlen += taosEncodeFixedI64(buf, param->delay); + } break; case TD_CHILD_TABLE: tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid); @@ -322,6 +337,20 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); } + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols); + for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]); + } + if (pReq->rollup && pReq->ntbCfg.pRSmaParam) { + SRSmaParam *param = pReq->ntbCfg.pRSmaParam; + tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor); + tlen += taosEncodeFixedI8(buf, param->delayUnit); + tlen += taosEncodeFixedI8(buf, param->nFuncIds); + for (int8_t i = 0; i < param->nFuncIds; ++i) { + tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); + } + tlen += taosEncodeFixedI64(buf, param->delay); + } break; default: ASSERT(0); @@ -332,16 +361,17 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI64(buf, &(pReq->ver)); + buf = taosDecodeString(buf, &(pReq->dbFName)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); - buf = taosDecodeFixedU8(buf, &(pReq->type)); + buf = taosDecodeFixedU8(buf, &(pReq->info)); switch (pReq->type) { case TD_SUPER_TABLE: buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid)); buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); - pReq->stbCfg.pSchema = (SSchema *)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); + pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); @@ -349,13 +379,39 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); } buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); - pReq->stbCfg.pTagSchema = (SSchema *)malloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); + pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } + buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); + if (pReq->stbCfg.nBSmaCols > 0) { + pReq->stbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); + for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { + buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i); + } + } else { + pReq->stbCfg.pBSmaCols = NULL; + } + if (pReq->rollup) { + pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); + SRSmaParam *param = pReq->stbCfg.pRSmaParam; + buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); + buf = taosDecodeFixedI8(buf, ¶m->delayUnit); + buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); + if (param->nFuncIds > 0) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { + buf = taosDecodeFixedI32(buf, param->pFuncIds + i); + } + } else { + param->pFuncIds = NULL; + } + buf = taosDecodeFixedI64(buf, ¶m->delay); + } else { + pReq->stbCfg.pRSmaParam = NULL; + } break; case TD_CHILD_TABLE: buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid); @@ -363,13 +419,39 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { break; case TD_NORMAL_TABLE: buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); - pReq->ntbCfg.pSchema = (SSchema *)malloc(pReq->ntbCfg.nCols * sizeof(SSchema)); + pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); } + buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); + if (pReq->ntbCfg.nBSmaCols > 0) { + pReq->ntbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t)); + for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { + buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i); + } + } else { + pReq->ntbCfg.pBSmaCols = NULL; + } + if (pReq->rollup) { + pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); + SRSmaParam *param = pReq->ntbCfg.pRSmaParam; + buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); + buf = taosDecodeFixedI8(buf, ¶m->delayUnit); + buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); + if (param->nFuncIds > 0) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { + buf = taosDecodeFixedI32(buf, param->pFuncIds + i); + } + } else { + param->pFuncIds = NULL; + } + buf = taosDecodeFixedI64(buf, ¶m->delay); + } else { + pReq->ntbCfg.pRSmaParam = NULL; + } break; default: ASSERT(0); @@ -398,7 +480,7 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { buf = taosDecodeFixedU32(buf, &nsize); pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq)); for (size_t i = 0; i < nsize; i++) { - SVCreateTbReq req; + SVCreateTbReq req = {0}; buf = tDeserializeSVCreateTbReq(buf, &req); taosArrayPush(pReq->pArray, &req); } @@ -428,8 +510,14 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1; + if (tEncodeI32(&encoder, pReq->aggregationMethod) < 0) return -1; + if (tEncodeI32(&encoder, pReq->delay) < 0) return -1; + if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; for (int32_t i = 0; i < pReq->numOfColumns; ++i) { SField *pField = taosArrayGet(pReq->pColumns, i); @@ -445,7 +533,16 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } - if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfSmas; ++i) { + SField *pField = taosArrayGet(pReq->pSmas, i); + if (tEncodeI8(&encoder, pField->type) < 0) return -1; + if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; + if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + } + + if (pReq->commentLen > 0) { + if (tEncodeBinary(&encoder, pReq->comment, pReq->commentLen) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -460,12 +557,19 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->aggregationMethod) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1; pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); - if (pReq->pColumns == NULL || pReq->pTags == NULL) { + pReq->pSmas = taosArrayInit(pReq->numOfSmas, sizeof(SField)); + if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pSmas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -492,7 +596,23 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR } } - if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfSmas; ++i) { + SField field = {0}; + if (tDecodeI8(&decoder, &field.type) < 0) return -1; + if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; + if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; + if (taosArrayPush(pReq->pSmas, &field) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + if (pReq->commentLen > 0) { + pReq->comment = taosMemoryMalloc(pReq->commentLen); + if (pReq->comment == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; + } + tEndDecode(&decoder); tCoderClear(&decoder); @@ -502,8 +622,11 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { taosArrayDestroy(pReq->pColumns); taosArrayDestroy(pReq->pTags); + taosArrayDestroy(pReq->pSmas); + taosMemoryFreeClear(pReq->comment); pReq->pColumns = NULL; pReq->pTags = NULL; + pReq->pSmas = NULL; } int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { @@ -589,6 +712,123 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) { pReq->pFields = NULL; } +int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->stb) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI8(&encoder, pReq->intervalUnit) < 0) return -1; + if (tEncodeI8(&encoder, pReq->slidingUnit) < 0) return -1; + if (tEncodeI8(&encoder, pReq->timezone) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dstVgId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->interval) < 0) return -1; + if (tEncodeI64(&encoder, pReq->offset) < 0) return -1; + if (tEncodeI64(&encoder, pReq->sliding) < 0) return -1; + if (tEncodeI32(&encoder, pReq->exprLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->tagsFilterLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->astLen) < 0) return -1; + if (pReq->exprLen > 0) { + if (tEncodeBinary(&encoder, pReq->expr, pReq->exprLen) < 0) return -1; + } + if (pReq->tagsFilterLen > 0) { + if (tEncodeBinary(&encoder, pReq->tagsFilter, pReq->tagsFilterLen) < 0) return -1; + } + if (pReq->sqlLen > 0) { + if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; + } + if (pReq->astLen > 0) { + if (tEncodeBinary(&encoder, pReq->ast, pReq->astLen) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->stb) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->intervalUnit) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->slidingUnit) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->timezone) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dstVgId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->interval) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->offset) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->sliding) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->exprLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->tagsFilterLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->astLen) < 0) return -1; + if (pReq->exprLen > 0) { + pReq->expr = taosMemoryMalloc(pReq->exprLen); + if (pReq->expr == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->expr) < 0) return -1; + } + if (pReq->tagsFilterLen > 0) { + pReq->tagsFilter = taosMemoryMalloc(pReq->tagsFilterLen); + if (pReq->tagsFilter == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->tagsFilter) < 0) return -1; + } + if (pReq->sqlLen > 0) { + pReq->sql = taosMemoryMalloc(pReq->sqlLen); + if (pReq->sql == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; + } + if (pReq->astLen > 0) { + pReq->ast = taosMemoryMalloc(pReq->astLen); + if (pReq->ast == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +void tFreeSMCreateSmaReq(SMCreateSmaReq *pReq) { + taosMemoryFreeClear(pReq->expr); + taosMemoryFreeClear(pReq->tagsFilter); + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); +} + +int32_t tSerializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -757,6 +997,8 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { return 0; } +void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); } + int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1302,6 +1544,14 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; if (tEncodeI8(&encoder, pReq->streamMode) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); + if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1335,12 +1585,36 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; if (tDecodeI8(&decoder, &pReq->streamMode) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; + pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention)); + if (pReq->pRetensions == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { + SRetention rentension = {0}; + if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; + if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + tEndDecode(&decoder); tCoderClear(&decoder); return 0; } +void tFreeSCreateDbReq(SCreateDbReq *pReq) { + taosArrayDestroy(pReq->pRetensions); + pReq->pRetensions = NULL; +} + int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1713,7 +1987,7 @@ int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; if (pReq->payloadLen > 0) { - pReq->payload = malloc(pReq->payloadLen); + pReq->payload = taosMemoryMalloc(pReq->payloadLen); if (pReq->payload == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; } @@ -1723,7 +1997,7 @@ int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { return 0; } -void tFreeSShowReq(SShowReq *pReq) { tfree(pReq->payload); } +void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); } int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq *pReq) { SCoder encoder = {0}; @@ -1731,7 +2005,10 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->type) < 0) return -1; if (tEncodeI8(&encoder, pReq->free) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1745,8 +2022,10 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; if (tDecodeI8(&decoder, &pReq->free) < 0) return -1; - + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); return 0; @@ -1792,7 +2071,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1; int32_t totalCols = pRsp->numOfTags + pRsp->numOfColumns; - pRsp->pSchemas = malloc(sizeof(SSchema) * totalCols); + pRsp->pSchemas = taosMemoryMalloc(sizeof(SSchema) * totalCols); if (pRsp->pSchemas == NULL) return -1; for (int32_t i = 0; i < totalCols; ++i) { @@ -1873,7 +2152,7 @@ int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatc return 0; } -void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { tfree(pRsp->pSchemas); } +void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); @@ -2025,13 +2304,13 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (sqlLen > 0) { - pReq->sql = calloc(1, sqlLen + 1); + pReq->sql = taosMemoryCalloc(1, sqlLen + 1); if (pReq->sql == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } if (astLen > 0) { - pReq->ast = calloc(1, astLen + 1); + pReq->ast = taosMemoryCalloc(1, astLen + 1); if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } @@ -2043,8 +2322,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR } void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { - tfree(pReq->sql); - tfree(pReq->ast); + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); } int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) { @@ -2213,6 +2492,14 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR SReplica *pReplica = &pReq->replicas[i]; if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; } + if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); + if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2257,11 +2544,35 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; } + if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; + pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention)); + if (pReq->pRetensions == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { + SRetention rentension = {0}; + if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; + if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + tEndDecode(&decoder); tCoderClear(&decoder); return 0; } +int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) { + taosArrayDestroy(pReq->pRetensions); + pReq->pRetensions = NULL; +} + int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2467,7 +2778,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - TCODER_MALLOC(pReq->offsets, SMqOffset*, pReq->num * sizeof(SMqOffset), decoder); + TCODER_MALLOC(pReq->offsets, SMqOffset *, pReq->num * sizeof(SMqOffset), decoder); if (pReq->offsets == NULL) return -1; for (int32_t i = 0; i < pReq->num; i++) { tDecodeSMqOffset(decoder, &pReq->offsets[i]); @@ -2560,7 +2871,6 @@ int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pR tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeU64(&encoder, pRsp->seqId) < 0) return -1; if (tEncodeI32(&encoder, pRsp->epId.nodeId) < 0) return -1; if (tEncodeU16(&encoder, pRsp->epId.ep.port) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->epId.ep.fqdn) < 0) return -1; @@ -2589,7 +2899,6 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp * tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeU64(&decoder, &pRsp->seqId) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->epId.nodeId) < 0) return -1; if (tDecodeU16(&decoder, &pRsp->epId.ep.port) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->epId.ep.fqdn) < 0) return -1; @@ -2617,6 +2926,77 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp * void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); } +int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->code) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (pRsp->rspList) { + int32_t num = taosArrayGetSize(pRsp->rspList); + if (tEncodeI32(&encoder, num) < 0) return -1; + for (int32_t i = 0; i < num; ++i) { + SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); + if (tEncodeI32(&encoder, rsp->code) < 0) return -1; + } + } else { + if (tEncodeI32(&encoder, 0) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { + SCoder decoder = {0}; + int32_t num = 0; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &num) < 0) return -1; + if (num > 0) { + pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp)); + if (NULL == pRsp->rspList) return -1; + for (int32_t i = 0; i < num; ++i) { + SVCreateTbRsp rsp = {0}; + if (tDecodeI32(&decoder, &rsp.code) < 0) return -1; + if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1; + } + } else { + pRsp->rspList = NULL; + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) { int32_t tlen = 0; @@ -2639,27 +3019,37 @@ int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->indexUid); tlen += taosEncodeString(buf, pReq->indexName); return tlen; } void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) { buf = taosDecodeFixedI64(buf, &(pReq->ver)); + buf = taosDecodeFixedI64(buf, &(pReq->indexUid)); buf = taosDecodeStringTo(buf, pReq->indexName); return buf; } int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) { + int32_t sqlLen = 0; + int32_t astLen = 0; + if (pReq->sql != NULL) sqlLen = (int32_t)strlen(pReq->sql); + if (pReq->ast != NULL) astLen = (int32_t)strlen(pReq->ast); + SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->outputSTbName) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1; + if (tEncodeI32(&encoder, sqlLen) < 0) return -1; + if (tEncodeI32(&encoder, astLen) < 0) return -1; + if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2668,15 +3058,30 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS } int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStreamReq *pReq) { + int32_t sqlLen = 0; + int32_t astLen = 0; + SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->outputSTbName) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; - if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; - if (tDecodeCStrAlloc(&decoder, &pReq->physicalPlan) < 0) return -1; - if (tDecodeCStrAlloc(&decoder, &pReq->logicalPlan) < 0) return -1; + if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; + if (tDecodeI32(&decoder, &astLen) < 0) return -1; + + if (sqlLen > 0) { + pReq->sql = taosMemoryCalloc(1, sqlLen + 1); + if (pReq->sql == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; + } + + if (astLen > 0) { + pReq->ast = taosMemoryCalloc(1, astLen + 1); + if (pReq->ast == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; + } tEndDecode(&decoder); tCoderClear(&decoder); @@ -2684,36 +3089,6 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea } void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { - tfree(pReq->sql); - tfree(pReq->physicalPlan); - tfree(pReq->logicalPlan); -} - -int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->level) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; - if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1; - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; - if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1; - tEndDecode(pDecoder); - return 0; -} - -void tFreeSStreamTask(SStreamTask *pTask) { - // TODO - /*free(pTask->qmsg);*/ - /*free(pTask->executor);*/ - /*free(pTask);*/ + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); } diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c new file mode 100644 index 0000000000000000000000000000000000000000..98ee1b679df579d07267ef8695a75361c60f6f39 --- /dev/null +++ b/source/common/src/tmsgcb.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "tmsgcb.h" + +int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { + return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq); +} + +int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { + return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype); +} + +int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq) { + return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq); +} + +int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) { + return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq); +} + +void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 370bf0396999d25a2ef019088a4c90ec3e7e3485..f4755f5b5e4e924ed0c917701430c97c4e08f5ef 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -30,13 +30,13 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil return NULL; } - SColumnFilterInfo* pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + SColumnFilterInfo* pFilter = taosMemoryCalloc(1, numOfFilters * sizeof(SColumnFilterInfo)); memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); for (int32_t j = 0; j < numOfFilters; ++j) { if (pFilter[j].filterstr) { size_t len = (size_t) pFilter[j].len + 1 * TSDB_NCHAR_SIZE; - pFilter[j].pz = (int64_t) calloc(1, len); + pFilter[j].pz = (int64_t) taosMemoryCalloc(1, len); memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t) pFilter[j].len); } @@ -171,7 +171,7 @@ bool tNameIsValid(const SName* name) { SName* tNameDup(const SName* name) { assert(name != NULL); - SName* p = malloc(sizeof(SName)); + SName* p = taosMemoryMalloc(sizeof(SName)); memcpy(p, name, sizeof(SName)); return p; } @@ -222,6 +222,27 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId) { return 0; } +bool tNameDBNameEqual(SName* left, SName* right) { + if (NULL == left) { + if (NULL == right) { + return true; + } + + return false; + } + + if (NULL == right) { + return false; + } + + if (left->acctId != right->acctId) { + return false; + } + + return (0 == strcmp(left->dbname, right->dbname)); +} + + int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { assert(dst != NULL && str != NULL && strlen(str) > 0); @@ -273,13 +294,4 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return 0; } -SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name) { - SSchema s = {0}; - s.type = type; - s.bytes = bytes; - s.colId = colId; - - tstrncpy(s.name, name, tListLen(s.name)); - return s; -} diff --git a/source/common/src/trow.c b/source/common/src/trow.c index db4bc49425e1835f3451bea18049f4488a1c2b42..9ee2ec1300487b5a46e3a63a2f8b8881b639b2ab 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -78,7 +78,7 @@ static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBit setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes); pCol->len += TYPE_BYTES[pCol->type]; } - if(setBitmap) { + if (setBitmap) { tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE); } } @@ -118,8 +118,8 @@ int trbWriteCol(SRowBuilder *pRB, void *pData, col_id_t cid) { #endif -STSRow* tdRowDup(STSRow *row) { - STSRow* trow = malloc(TD_ROW_LEN(row)); +STSRow *tdRowDup(STSRow *row) { + STSRow *trow = taosMemoryMalloc(TD_ROW_LEN(row)); if (trow == NULL) return NULL; tdRowCpy(trow, row); @@ -176,7 +176,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); } while (dcol < pCols->numOfCols) { @@ -378,9 +378,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i } } - - -STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) { +STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) { #if 0 ASSERT(TD_ROW_KEY(row1) == TD_ROW_KEY(row2)); ASSERT(schemaVersion(pSchema1) == TD_ROW_SVER(row1)); @@ -473,6 +471,44 @@ STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema } taosArrayDestroy(stashRow); return buffer; - #endif +#endif return NULL; } + +SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { + SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); + if (pRet == NULL) return NULL; + + pRet->numOfCols = pDataCols->numOfCols; + pRet->sversion = pDataCols->sversion; + if (keepData) pRet->numOfRows = pDataCols->numOfRows; + + for (int i = 0; i < pDataCols->numOfCols; i++) { + pRet->cols[i].type = pDataCols->cols[i].type; + pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; + pRet->cols[i].colId = pDataCols->cols[i].colId; + pRet->cols[i].bytes = pDataCols->cols[i].bytes; + pRet->cols[i].offset = pDataCols->cols[i].offset; + + if (keepData) { + if (pDataCols->cols[i].len > 0) { + if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { + tdFreeDataCols(pRet); + return NULL; + } + pRet->cols[i].len = pDataCols->cols[i].len; + memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { + int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints; + memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); + } + if (!TD_COL_ROWS_NORM(pRet->cols + i)) { + int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints); + memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); + } + } + } + } + + return pRet; +} \ No newline at end of file diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 22c10f62b4ee11df24af06b8131b4dde89e9be27..510a2809e74b92acecc510b928d9fe288ba9f7db 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -458,16 +458,21 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; } - if (unit == 'y') { - duration *= 12; - } else if (unit != 'n') { + + if (unit != 'n' && unit != 'y') { return t + duration; } + // The following code handles the y/n time duration + int64_t numOfMonth = duration; + if (unit == 'y') { + numOfMonth *= 12; + } + struct tm tm; time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); taosLocalTime(&tt, &tm); - int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)duration; + int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -557,8 +562,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // not enough time range if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = start + pInterval->interval - 1; - + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; while (end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window start += pInterval->sliding; @@ -574,12 +578,23 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio } } + ASSERT(pInterval->offset >= 0); + if (pInterval->offset > 0) { start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); if (start > t) { start = taosTimeAdd(start, -pInterval->interval, pInterval->intervalUnit, precision); + } else { + // try to move current window to the left-hande-side, due to the offset effect. + int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + ASSERT(end >= t); + end = taosTimeAdd(end, -pInterval->sliding, pInterval->slidingUnit, precision); + if (end >= t) { + start = taosTimeAdd(start, -pInterval->sliding, pInterval->slidingUnit, precision); + } } } + return start; } diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 5f0a3532268f3222d3852d5b31e7f999fe307ff9..15e741d3076161f33738124dadfaad551bb56bca 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -30,7 +30,7 @@ static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); * @return */ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); + STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); if (pTSBuf == NULL) { return NULL; } @@ -41,7 +41,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (pTSBuf->pFile == NULL) { - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -66,7 +66,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); + STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); if (pTSBuf == NULL) { return NULL; } @@ -78,7 +78,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); if (pTSBuf->pFile == NULL) { - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -101,7 +101,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { if (header.numOfGroup > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfGroup; - STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); + STSGroupBlockInfoEx* tmp = taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -122,7 +122,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); + STSGroupBlockInfo* buf = (STSGroupBlockInfo*)taosMemoryCalloc(1, infoSize); if (buf == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -137,7 +137,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); } - free(buf); + taosMemoryFree(buf); ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); @@ -166,11 +166,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) { return NULL; } - tfree(pTSBuf->assistBuf); - tfree(pTSBuf->tsData.rawBuf); + taosMemoryFreeClear(pTSBuf->assistBuf); + taosMemoryFreeClear(pTSBuf->tsData.rawBuf); - tfree(pTSBuf->pData); - tfree(pTSBuf->block.payload); + taosMemoryFreeClear(pTSBuf->pData); + taosMemoryFreeClear(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { taosCloseFile(&pTSBuf->pFile); @@ -184,7 +184,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { } taosVariantDestroy(&pTSBuf->block.tag); - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -200,7 +200,7 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); assert((int32_t)newSize > pTSBuf->numOfAlloc); - STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return NULL; } @@ -240,7 +240,7 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { static void shrinkBuffer(STSList* ptsData) { // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size if (ptsData->allocSize >= ptsData->threshold * 2) { - char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE); + char* rawBuf = taosMemoryRealloc(ptsData->rawBuf, MEM_BUF_SIZE); if (rawBuf) { ptsData->rawBuf = rawBuf; ptsData->allocSize = MEM_BUF_SIZE; @@ -322,7 +322,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { static void expandBuffer(STSList* ptsData, int32_t inputSize) { if (ptsData->allocSize - ptsData->len < inputSize) { int32_t newSize = inputSize + ptsData->len; - char* tmp = realloc(ptsData->rawBuf, (size_t)newSize); + char* tmp = taosMemoryRealloc(ptsData->rawBuf, (size_t)newSize); if (tmp == NULL) { // todo } @@ -366,7 +366,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // NOTE: mix types tags are not supported size_t sz = 0; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); + char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1); assert(tp != NULL); memset(tp, 0, pBlock->tag.nLen + 1); @@ -812,7 +812,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { if (pDestBuf->numOfAlloc < newSize) { pDestBuf->numOfAlloc = newSize; - STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = taosMemoryRealloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return -1; } @@ -1028,13 +1028,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { const int32_t INITIAL_GROUPINFO_SIZE = 4; pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; - pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); + pTSBuf->pData = taosMemoryCalloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); if (pTSBuf->pData == NULL) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); + pTSBuf->tsData.rawBuf = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->tsData.rawBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -1044,13 +1044,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { pTSBuf->tsData.threshold = MEM_BUF_SIZE; pTSBuf->tsData.allocSize = MEM_BUF_SIZE; - pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); + pTSBuf->assistBuf = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->assistBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->block.payload = malloc(MEM_BUF_SIZE); + pTSBuf->block.payload = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->block.payload == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -1079,7 +1079,7 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { return; } - (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); + (*id) = taosMemoryMalloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); for (int32_t i = 0; i < size; ++i) { (*id)[i] = pTSBuf->pData[i].info.id; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index c6aa1cb81d7a7fe1c48f90f37218a14e63f7cf07..3995db89b615fa19a36924659e0af79cb78afdbc 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -199,14 +199,14 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; - pVar->ucs4 = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); + pVar->ucs4 = taosMemoryCalloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); memcpy(pVar->ucs4, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method - pVar->pz = calloc(len + 1, sizeof(char)); + pVar->pz = taosMemoryCalloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); pVar->nLen = (int32_t)len; break; @@ -224,7 +224,7 @@ void taosVariantDestroy(SVariant *pVar) { if (pVar == NULL) return; if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) { - tfree(pVar->pz); + taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } @@ -233,7 +233,7 @@ void taosVariantDestroy(SVariant *pVar) { size_t num = taosArrayGetSize(pVar->arr); for (size_t i = 0; i < num; i++) { void *p = taosArrayGetP(pVar->arr, i); - free(p); + taosMemoryFree(p); } taosArrayDestroy(pVar->arr); pVar->arr = NULL; @@ -254,7 +254,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; - char *p = realloc(pDst->pz, len); + char *p = taosMemoryRealloc(pDst->pz, len); assert(p); memset(p, 0, len); @@ -402,18 +402,18 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { // it is a in-place convert type for SVariant, local buffer is needed if (*pDest == pVariant->pz) { - pBuf = calloc(1, INITIAL_ALLOC_SIZE); + pBuf = taosMemoryCalloc(1, INITIAL_ALLOC_SIZE); } if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { size_t newSize = pVariant->nLen * TSDB_NCHAR_SIZE; if (pBuf != NULL) { if (newSize >= INITIAL_ALLOC_SIZE) { - pBuf = realloc(pBuf, newSize + 1); + pBuf = taosMemoryRealloc(pBuf, newSize + 1); } taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, pBuf); - free(pVariant->ucs4); + taosMemoryFree(pVariant->ucs4); pBuf[newSize] = 0; } else { taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, *pDest); @@ -460,23 +460,23 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } if (*pDest == pVariant->pz) { - TdUcs4 *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); + TdUcs4 *pWStr = taosMemoryCalloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); bool ret = taosMbsToUcs4(pDst, nLen, pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); if (!ret) { - tfree(pWStr); + taosMemoryFreeClear(pWStr); return -1; } // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { - free(pVariant->ucs4); + taosMemoryFree(pVariant->ucs4); } pVariant->ucs4 = pWStr; *pDestSize = taosUcs4len(pVariant->ucs4); // shrink the allocate memory, no need to check here. - char *tmp = realloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE); + char *tmp = taosMemoryRealloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE); assert(tmp != NULL); pVariant->ucs4 = (TdUcs4 *)tmp; @@ -526,7 +526,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result if (token.type == TK_NULL) { if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -547,7 +547,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -566,7 +566,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result if (token.type == TK_FLOAT) { double v = wcstod(pVariant->ucs4, &endPtr); if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -577,7 +577,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result *result = (int64_t)v; } else if (token.type == TK_NULL) { if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } setNull((char *)result, type, tDataTypes[type].bytes); @@ -585,7 +585,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } else { int64_t val = wcstoll(pVariant->ucs4, &endPtr, 10); if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -971,21 +971,21 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { errno = 0; double v = strtod(pVariant->pz, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); return -1; } - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->d = v; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { errno = 0; double v = wcstod(pVariant->ucs4, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); return -1; } - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->d = v; } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { double tmp = (double)pVariant->i; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index d5fb61929a94109331e955e492cb3333b09c961a..ccd800d3f4686d70ffcc6da8c9951170cdd2aa03 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -99,7 +99,7 @@ TEST(testCase, toInteger_test) { } TEST(testCase, Datablock_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -108,8 +108,8 @@ TEST(testCase, Datablock_test) { infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.colId = 1; - infoData.pData = (char*) calloc(40, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (40/8)); + infoData.pData = (char*) taosMemoryCalloc(40, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (40/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -117,7 +117,7 @@ TEST(testCase, Datablock_test) { infoData1.info.type = TSDB_DATA_TYPE_BINARY; infoData1.info.colId = 2; - infoData1.varmeta.offset = (int32_t*) calloc(40, sizeof(uint32_t)); + infoData1.varmeta.offset = (int32_t*) taosMemoryCalloc(40, sizeof(uint32_t)); taosArrayPush(b->pDataBlock, &infoData1); char* str = "the value of: %d"; @@ -178,7 +178,7 @@ TEST(testCase, Datablock_test) { #if 0 TEST(testCase, non_var_dataBlock_split_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -189,8 +189,8 @@ TEST(testCase, non_var_dataBlock_split_test) { int32_t numOfRows = 1000000; - infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -198,8 +198,8 @@ TEST(testCase, non_var_dataBlock_split_test) { infoData1.info.type = TSDB_DATA_TYPE_TINYINT; infoData1.info.colId = 2; - infoData1.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData1.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData1.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData1.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData1); for(int32_t i = 0; i < numOfRows; ++i) { @@ -233,7 +233,7 @@ TEST(testCase, non_var_dataBlock_split_test) { #endif TEST(testCase, var_dataBlock_split_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -244,8 +244,8 @@ TEST(testCase, var_dataBlock_split_test) { infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.colId = 1; - infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -253,7 +253,7 @@ TEST(testCase, var_dataBlock_split_test) { infoData1.info.type = TSDB_DATA_TYPE_BINARY; infoData1.info.colId = 2; - infoData1.varmeta.offset = (int32_t*) calloc(numOfRows, sizeof(uint32_t)); + infoData1.varmeta.offset = (int32_t*) taosMemoryCalloc(numOfRows, sizeof(uint32_t)); taosArrayPush(b->pDataBlock, &infoData1); char buf[41] = {0}; diff --git a/source/dnode/bnode/inc/bndInt.h b/source/dnode/bnode/inc/bndInt.h index 4a0d16a7498c1166b3d4742111cf7b2f676abbcc..e611d230a329ad597ba5b7ee634c3246b6c44294 100644 --- a/source/dnode/bnode/inc/bndInt.h +++ b/source/dnode/bnode/inc/bndInt.h @@ -30,7 +30,7 @@ extern "C" { #endif typedef struct SBnode { - SBnodeOpt opt; + SMsgCb msgCb; } SBnode; #ifdef __cplusplus diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c index 9570bc72a0e620f79a2f50e41ab14408be90e7af..b9c41ebf4370fb070a39a7d76dab61742a174cf6 100644 --- a/source/dnode/bnode/src/bnode.c +++ b/source/dnode/bnode/src/bnode.c @@ -16,14 +16,13 @@ #include "bndInt.h" SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) { - SBnode *pBnode = calloc(1, sizeof(SBnode)); + SBnode *pBnode = taosMemoryCalloc(1, sizeof(SBnode)); + pBnode->msgCb = pOption->msgCb; return pBnode; } -void bndClose(SBnode *pBnode) { free(pBnode); } +void bndClose(SBnode *pBnode) { taosMemoryFree(pBnode); } int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; } - -void bndDestroy(const char *path) {} diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 64e8980219aa2c0043986898a98e16cc561b8c27..35ea16698378da156fa2caec708944bdef0c1336 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,2 +1,30 @@ -add_subdirectory(daemon) -add_subdirectory(impl) \ No newline at end of file +aux_source_directory(src DNODE_SRC) +aux_source_directory(dnode/src DNODE_SRC) +aux_source_directory(qnode/src DNODE_SRC) +aux_source_directory(bnode/src DNODE_SRC) +aux_source_directory(snode/src DNODE_SRC) +aux_source_directory(vnode/src DNODE_SRC) +aux_source_directory(mnode/src DNODE_SRC) +aux_source_directory(container/src DNODE_SRC) + +add_library(dnode STATIC ${DNODE_SRC}) +target_link_libraries( + dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor +) +target_include_directories( + dnode + PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/snode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/container/inc" +) + +add_subdirectory(main) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/dnode/mgmt/impl/inc/dndBnode.h b/source/dnode/mgmt/bnode/inc/bm.h similarity index 68% rename from source/dnode/mgmt/impl/inc/dndBnode.h rename to source/dnode/mgmt/bnode/inc/bm.h index 080cd2e487c2ca74bb3151742b4eed91a28040b3..79cf76d11379be81c1b094806b4fadcb97553762 100644 --- a/source/dnode/mgmt/impl/inc/dndBnode.h +++ b/source/dnode/mgmt/bnode/inc/bm.h @@ -16,17 +16,13 @@ #ifndef _TD_DND_BNODE_H_ #define _TD_DND_BNODE_H_ +#include "dnd.h" + #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" - -int32_t dndInitBnode(SDnode *pDnode); -void dndCleanupBnode(SDnode *pDnode); -void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +void bmGetMgmtFp(SMgmtWrapper *pWrapper); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/bnode/inc/bmInt.h b/source/dnode/mgmt/bnode/inc/bmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..8cfff0f1f3f0c9ad56a4d0d3bbf332a1b4d13c3c --- /dev/null +++ b/source/dnode/mgmt/bnode/inc/bmInt.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_BNODE_INT_H_ +#define _TD_DND_BNODE_INT_H_ + +#include "bm.h" +#include "bnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SBnodeMgmt { + SBnode *pBnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SMultiWorker writeWorker; +} SBnodeMgmt; + +// bmInt.c +int32_t bmOpen(SMgmtWrapper *pWrapper); +int32_t bmDrop(SMgmtWrapper *pWrapper); + +// bmMsg.c +void bmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// bmWorker.c +int32_t bmStartWorker(SBnodeMgmt *pMgmt); +void bmStopWorker(SBnodeMgmt *pMgmt); +int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_BNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/bnode/src/bmInt.c b/source/dnode/mgmt/bnode/src/bmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..e2506ab38322845918c269c66bbdddcb3c8c48d8 --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmInt.c @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + pOption->msgCb = msgCb; +} + +static int32_t bmOpenImp(SBnodeMgmt *pMgmt) { + SBnodeOpt option = {0}; + bmInitOption(pMgmt, &option); + + pMgmt->pBnode = bndOpen(pMgmt->path, &option); + if (pMgmt->pBnode == NULL) { + dError("failed to open bnode since %s", terrstr()); + return -1; + } + + if (bmStartWorker(pMgmt) != 0) { + dError("failed to start bnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write bnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void bmCloseImp(SBnodeMgmt *pMgmt) { + if (pMgmt->pBnode != NULL) { + bmStopWorker(pMgmt); + bndClose(pMgmt->pBnode); + pMgmt->pBnode = NULL; + } +} + +int32_t bmDrop(SMgmtWrapper *pWrapper) { + SBnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("bnode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop bnode since %s", terrstr()); + return -1; + } + + bmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + taosMemoryFree(pMgmt); + dInfo("bnode-mgmt is dropped"); + return 0; +} + +static void bmClose(SMgmtWrapper *pWrapper) { + SBnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("bnode-mgmt start to cleanup"); + bmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + taosMemoryFree(pMgmt); + dInfo("bnode-mgmt is cleaned up"); +} + +int32_t bmOpen(SMgmtWrapper *pWrapper) { + dInfo("bnode-mgmt start to init"); + SBnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SBnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = bmOpenImp(pMgmt); + if (code != 0) { + dError("failed to init bnode-mgmt since %s", terrstr()); + bmClose(pWrapper); + } else { + dInfo("bnode-mgmt is initialized"); + } + + return code; +} + +void bmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = bmOpen; + mgmtFp.closeFp = bmClose; + mgmtFp.createMsgFp = bmProcessCreateReq; + mgmtFp.dropMsgFp = bmProcessDropReq; + mgmtFp.requiredFp = bmRequire; + + bmInitMsgHandles(pWrapper); + pWrapper->name = "bnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/bnode/src/bmMsg.c b/source/dnode/mgmt/bnode/src/bmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..c01d260c3f1c1c2fae5045765fd953bc15e1f07a --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmMsg.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateBnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return bmOpen(pWrapper); + } +} + +int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropBnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop bnode since %s", terrstr()); + return -1; + } else { + return bmDrop(pWrapper); + } +} + +void bmInitMsgHandles(SMgmtWrapper *pWrapper) {} diff --git a/source/dnode/mgmt/bnode/src/bmWorker.c b/source/dnode/mgmt/bnode/src/bmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..7698aa9dbdaaa2f8adbd24b9db2115dd142f16b9 --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmWorker.c @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; + dndSendRsp(pWrapper, &rpcRsp); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t numOfMsgs, int32_t code) { + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + bmSendErrorRsp(pWrapper, pMsg, code); + } +} + +static void bmProcessQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SBnodeMgmt *pMgmt = pInfo->ahandle; + SMgmtWrapper *pWrapper = pMgmt->pWrapper; + + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); + if (pArray == NULL) { + bmSendErrorRsps(pWrapper, qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); + return; + } + + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + dTrace("msg:%p, will be processed in bnode queue", pMsg); + if (taosArrayPush(pArray, &pMsg) == NULL) { + bmSendErrorRsp(pWrapper, pMsg, TSDB_CODE_OUT_OF_MEMORY); + } + } + + bndProcessWMsgs(pMgmt->pBnode, pArray); + + for (size_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } + taosArrayDestroy(pArray); +} + +int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SMultiWorker *pWorker = &pMgmt->writeWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return taosWriteQitem(pWorker->queue, pMsg); +} + +int32_t bmStartWorker(SBnodeMgmt *pMgmt) { + SMultiWorkerCfg cfg = {.maxNum = 1, .name = "bnode-write", .fp = (FItems)bmProcessQueue, .param = pMgmt}; + if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) { + dError("failed to start bnode write worker since %s", terrstr()); + return -1; + } + + dDebug("bnode workers are initialized"); + return 0; +} + +void bmStopWorker(SBnodeMgmt *pMgmt) { + tMultiWorkerCleanup(&pMgmt->writeWorker); + dDebug("bnode workers are closed"); +} diff --git a/source/dnode/mgmt/container/inc/dnd.h b/source/dnode/mgmt/container/inc/dnd.h new file mode 100644 index 0000000000000000000000000000000000000000..f6c8897f643c32d96c62334b673b4e4bdfe4d6b1 --- /dev/null +++ b/source/dnode/mgmt/container/inc/dnd.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_H_ +#define _TD_DND_H_ + +#include "os.h" + +#include "cJSON.h" +#include "tcache.h" +#include "tcrc32c.h" +#include "tdatablock.h" +#include "tglobal.h" +#include "thash.h" +#include "tlockfree.h" +#include "tlog.h" +#include "tmsg.h" +#include "tprocess.h" +#include "tqueue.h" +#include "trpc.h" +#include "tthread.h" +#include "ttime.h" +#include "tworker.h" + +#include "dnode.h" +#include "monitor.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} + +typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; +typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; +typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus; +typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType; + +typedef struct SMgmtFp SMgmtFp; +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SMsgHandle SMsgHandle; +typedef struct SDnodeMgmt SDnodeMgmt; +typedef struct SVnodesMgmt SVnodesMgmt; +typedef struct SMnodeMgmt SMnodeMgmt; +typedef struct SQnodeMgmt SQnodeMgmt; +typedef struct SSnodeMgmt SSnodeMgmt; +typedef struct SBnodeMgmt SBnodeMgmt; + +typedef int32_t (*NodeMsgFp)(void *pMgmt, SNodeMsg *pMsg); +typedef int32_t (*OpenNodeFp)(SMgmtWrapper *pWrapper); +typedef void (*CloseNodeFp)(SMgmtWrapper *pWrapper); +typedef int32_t (*StartNodeFp)(SMgmtWrapper *pWrapper); +typedef int32_t (*CreateNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); + +typedef struct SMsgHandle { + SMgmtWrapper *pQndWrapper; + SMgmtWrapper *pMndWrapper; + SMgmtWrapper *pWrapper; +} SMsgHandle; + +typedef struct SMgmtFp { + OpenNodeFp openFp; + CloseNodeFp closeFp; + StartNodeFp startFp; + CreateNodeFp createMsgFp; + DropNodeFp dropMsgFp; + RequireNodeFp requiredFp; +} SMgmtFp; + +typedef struct SMgmtWrapper { + const char *name; + char *path; + int32_t refCount; + SRWLatch latch; + bool deployed; + bool required; + EProcType procType; + SProcObj *pProc; + void *pMgmt; + SDnode *pDnode; + NodeMsgFp msgFps[TDMT_MAX]; + int32_t msgVgIds[TDMT_MAX]; // Handle the case where the same message type is distributed to qnode or vnode + SMgmtFp fp; +} SMgmtWrapper; + +typedef struct { + void *serverRpc; + void *clientRpc; + SMsgHandle msgHandles[TDMT_MAX]; +} STransMgmt; + +typedef struct SDnode { + int64_t clusterId; + int32_t dnodeId; + int32_t numOfSupportVnodes; + int64_t rebootTime; + char *localEp; + char *localFqdn; + char *firstEp; + char *secondEp; + char *dataDir; + SDiskCfg *pDisks; + int32_t numOfDisks; + uint16_t serverPort; + bool dropped; + EDndStatus status; + EDndEvent event; + SStartupReq startup; + TdFilePtr pLockFile; + STransMgmt trans; + SMgmtWrapper wrappers[NODE_MAX]; +} SDnode; + +EDndStatus dndGetStatus(SDnode *pDnode); +void dndSetStatus(SDnode *pDnode, EDndStatus stat); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId); +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); +void dndSendMonitorReport(SDnode *pDnode); + +int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg); +void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); + +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); + +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); +void dndReleaseWrapper(SMgmtWrapper *pWrapper); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/container/inc/dndInt.h b/source/dnode/mgmt/container/inc/dndInt.h new file mode 100644 index 0000000000000000000000000000000000000000..d10835b67f760dce989290e806d815e8d83e18a5 --- /dev/null +++ b/source/dnode/mgmt/container/inc/dndInt.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_INT_H_ +#define _TD_DND_INT_H_ + +#include "dnd.h" + +#include "bm.h" +#include "dm.h" +#include "mm.h" +#include "qm.h" +#include "sm.h" +#include "vm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// dndInt.c +int32_t dndInit(); +void dndCleanup(); +const char *dndStatStr(EDndStatus stat); +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); +TdFilePtr dndCheckRunning(const char *dataDir); +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); + +// dndMsg.c +void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); + +// dndExec.c +int32_t dndOpenNode(SMgmtWrapper *pWrapper); +void dndCloseNode(SMgmtWrapper *pWrapper); +int32_t dndRun(SDnode *pDnode); + +// dndObj.c +SDnode *dndCreate(const SDnodeOpt *pOption); +void dndClose(SDnode *pDnode); +void dndHandleEvent(SDnode *pDnode, EDndEvent event); + +// dndTransport.c +int32_t dndInitServer(SDnode *pDnode); +void dndCleanupServer(SDnode *pDnode); +int32_t dndInitClient(SDnode *pDnode); +void dndCleanupClient(SDnode *pDnode); +int32_t dndInitMsgHandle(SDnode *pDnode); +void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c new file mode 100644 index 0000000000000000000000000000000000000000..3c29f80c94f260156bbb2badfd096a9361faa6d8 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static void dndResetLog(SMgmtWrapper *pMgmt) { + char logname[24] = {0}; + snprintf(logname, sizeof(logname), "%slog", pMgmt->name); + + dInfo("node:%s, reset log to %s", pMgmt->name, logname); + taosCloseLog(); + taosInitLog(logname, 1); +} + +static bool dndRequireNode(SMgmtWrapper *pWrapper) { + bool required = false; + int32_t code =(*pWrapper->fp.requiredFp)(pWrapper, &required); + if (!required) { + dDebug("node:%s, no need to start", pWrapper->name); + } else { + dDebug("node:%s, need to start", pWrapper->name); + } + return required; +} + +int32_t dndOpenNode(SMgmtWrapper *pWrapper) { + int32_t code = (*pWrapper->fp.openFp)(pWrapper); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + return -1; + } else { + dDebug("node:%s, has been opened", pWrapper->name); + } + + pWrapper->deployed = true; + return 0; +} + +void dndCloseNode(SMgmtWrapper *pWrapper) { + dDebug("node:%s, start to close", pWrapper->name); + taosWLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + (*pWrapper->fp.closeFp)(pWrapper); + pWrapper->deployed = false; + } + taosWUnLockLatch(&pWrapper->latch); + + while (pWrapper->refCount > 0) { + taosMsleep(10); + } + + if (pWrapper->pProc) { + taosProcCleanup(pWrapper->pProc); + pWrapper->pProc = NULL; + } + dDebug("node:%s, has been closed", pWrapper->name); +} + +static int32_t dndRunInSingleProcess(SDnode *pDnode) { + dInfo("dnode run in single process mode"); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + pWrapper->required = dndRequireNode(pWrapper); + if (!pWrapper->required) continue; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + dInfo("node:%s, will start in single process", pWrapper->name); + pWrapper->procType = PROC_SINGLE; + if (dndOpenNode(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + } + + dndSetStatus(pDnode, DND_STAT_RUNNING); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + if (!pWrapper->required) continue; + if (pWrapper->fp.startFp == NULL) continue; + if ((*pWrapper->fp.startFp)(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + } + + return 0; +} + +static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) { + dndCleanupServer(pDnode); + for (ENodeType n = 0; n < NODE_MAX; ++n) { + if (except == n) continue; + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + dndCloseNode(pWrapper); + } +} + +static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) { + dTrace("msg:%p, get from child queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + pRpc->pCont = pCont; + + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + int32_t code = (*msgFp)(pWrapper, pMsg); + + if (code != 0) { + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pCont); + } +} + +static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { + dTrace("msg:%p, get from parent queue", pRsp); + pRsp->pCont = pCont; + dndSendRpcRsp(pWrapper, pRsp); + taosMemoryFree(pRsp); +} + +static int32_t dndRunInMultiProcess(SDnode *pDnode) { + dInfo("dnode run in multi process mode"); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + pWrapper->required = dndRequireNode(pWrapper); + if (!pWrapper->required) continue; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + if (n == DNODE) { + dInfo("node:%s, will start in parent process", pWrapper->name); + pWrapper->procType = PROC_SINGLE; + if (dndOpenNode(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + continue; + } + + SProcCfg cfg = {.childQueueSize = 1024 * 1024 * 2, // size will be a configuration item + .childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentQueueSize = 1024 * 1024 * 2, // size will be a configuration item + .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, + .parentdMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .testFlag = 0, + .pParent = pWrapper, + .name = pWrapper->name}; + SProcObj *pProc = taosProcInit(&cfg); + if (pProc == NULL) { + dError("node:%s, failed to fork since %s", pWrapper->name, terrstr()); + return -1; + } + + pWrapper->pProc = pProc; + + if (taosProcIsChild(pProc)) { + dInfo("node:%s, will start in child process", pWrapper->name); + pWrapper->procType = PROC_CHILD; + dndResetLog(pWrapper); + + dInfo("node:%s, clean up resources inherited from parent", pWrapper->name); + dndClearNodesExecpt(pDnode, n); + + dInfo("node:%s, will be initialized in child process", pWrapper->name); + dndOpenNode(pWrapper); + } else { + dInfo("node:%s, will not start in parent process", pWrapper->name); + pWrapper->procType = PROC_PARENT; + } + + if (taosProcRun(pProc) != 0) { + dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); + return -1; + } + } + +#if 0 + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper->procType == PROC_PARENT && dmStart(pWrapper->pMgmt) != 0) { + dndReleaseWrapper(pWrapper); + dError("failed to start dnode worker since %s", terrstr()); + return -1; + } + + dndReleaseWrapper(pWrapper); +#endif + return 0; +} + +int32_t dndRun(SDnode *pDnode) { + if (tsMultiProcess == 0) { + if (dndRunInSingleProcess(pDnode) != 0) { + dError("failed to run dnode in single process mode since %s", terrstr()); + return -1; + } + } else { + if (dndRunInMultiProcess(pDnode) != 0) { + dError("failed to run dnode in multi process mode since %s", terrstr()); + return -1; + } + } + + dndReportStartup(pDnode, "TDengine", "initialized successfully"); + + while (1) { + if (pDnode->event == DND_EVENT_STOP) { + dInfo("dnode is about to stop"); + break; + } + taosMsleep(100); + } + + return 0; +} diff --git a/source/dnode/mgmt/container/src/dndFile.c b/source/dnode/mgmt/container/src/dndFile.c new file mode 100644 index 0000000000000000000000000000000000000000..f860bf8e2de455f1e423965a74fe8c9f22bceb78 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndFile.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 1024; + char *content = taosMemoryCalloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto _OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto _OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto _OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", file); + goto _OVER; + } + *pDeployed = deployed->valueint != 0; + + code = 0; + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + +_OVER: + if (content != NULL) taosMemoryFree(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 1024; + char *content = taosMemoryCalloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + taosMemoryFree(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d", realfile, deployed); + return 0; +} diff --git a/source/dnode/mgmt/container/src/dndInt.c b/source/dnode/mgmt/container/src/dndInt.c new file mode 100644 index 0000000000000000000000000000000000000000..33d6bb0ee33822a0ce758b4a1887047e6a711c73 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndInt.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" +#include "wal.h" + +static int8_t once = DND_ENV_INIT; + +int32_t dndInit() { + dInfo("start to init dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { + terrno = TSDB_CODE_REPEAT_INIT; + dError("failed to init dnode env since %s", terrstr()); + return -1; + } + + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + + if (rpcInit() != 0) { + dError("failed to init rpc since %s", terrstr()); + dndCleanup(); + return -1; + } + + SMonCfg monCfg = {0}; + monCfg.maxLogs = tsMonitorMaxLogs; + monCfg.port = tsMonitorPort; + monCfg.server = tsMonitorFqdn; + monCfg.comp = tsMonitorComp; + if (monInit(&monCfg) != 0) { + dError("failed to init monitor since %s", terrstr()); + dndCleanup(); + return -1; + } + + dInfo("dnode env is initialized"); + return 0; +} + +void dndCleanup() { + dInfo("start to cleanup dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + dError("dnode env is already cleaned up"); + return; + } + + monCleanup(); + rpcCleanup(); + walCleanUp(); + taosStopCacheRefreshWorker(); + dInfo("dnode env is cleaned up"); +} + +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId) { + pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; + pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; +} + +EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } + +void dndSetStatus(SDnode *pDnode, EDndStatus status) { + if (pDnode->status != status) { + dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); + pDnode->status = status; + } +} + +const char *dndStatStr(EDndStatus status) { + switch (status) { + case DND_STAT_INIT: + return "init"; + case DND_STAT_RUNNING: + return "running"; + case DND_STAT_STOPPED: + return "stopped"; + default: + return "unknown"; + } +} + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} + +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +TdFilePtr dndCheckRunning(const char *dataDir) { + char filepath[PATH_MAX] = {0}; + snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s", filepath, terrstr()); + return NULL; + } + + int32_t ret = taosLockFile(pFile); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s", filepath, terrstr()); + taosCloseFile(&pFile); + return NULL; + } + + dDebug("file:%s is locked", filepath); + return pFile; +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + SRpcMsg rpcRsp = { + .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; + rpcSendResponse(&rpcRsp); +} diff --git a/source/dnode/mgmt/container/src/dndMonitor.c b/source/dnode/mgmt/container/src/dndMonitor.c new file mode 100644 index 0000000000000000000000000000000000000000..c01f8407945ec3efa4193f9fb004dded3f559abb --- /dev/null +++ b/source/dnode/mgmt/container/src/dndMonitor.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + vmMonitorTfsInfo(pWrapper, pInfo); + dndReleaseWrapper(pWrapper); + } + return 0; +} + +static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { + pInfo->protocol = 1; + pInfo->dnode_id = pDnode->dnodeId; + pInfo->cluster_id = pDnode->clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + +static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { + pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f); + taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); + taosGetCpuCores(&pInfo->cpu_cores); + taosGetProcMemory(&pInfo->mem_engine); + taosGetSysMemory(&pInfo->mem_system); + pInfo->mem_total = tsTotalMemoryKB; + pInfo->disk_engine = 0; + pInfo->disk_used = tsDataSpace.size.used; + pInfo->disk_total = tsDataSpace.size.total; + taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); + taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + vmMonitorVnodeReqs(pWrapper, pInfo); + dndReleaseWrapper(pWrapper); + } + + pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + pInfo->has_mnode = pWrapper->required; + dndReleaseWrapper(pWrapper); + } +} + +void dndSendMonitorReport(SDnode *pDnode) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + SMonBasicInfo basicInfo = {0}; + dndGetMonitorBasicInfo(pDnode, &basicInfo); + monSetBasicInfo(pMonitor, &basicInfo); + + SMonClusterInfo clusterInfo = {0}; + SMonVgroupInfo vgroupInfo = {0}; + SMonGrantInfo grantInfo = {0}; + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + if (mmMonitorMnodeInfo(pWrapper, &clusterInfo, &vgroupInfo, &grantInfo) == 0) { + monSetClusterInfo(pMonitor, &clusterInfo); + monSetVgroupInfo(pMonitor, &vgroupInfo); + monSetGrantInfo(pMonitor, &grantInfo); + } + dndReleaseWrapper(pWrapper); + } + + SMonDnodeInfo dnodeInfo = {0}; + dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); + monSetDnodeInfo(pMonitor, &dnodeInfo); + + SMonDiskInfo diskInfo = {0}; + if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { + monSetDiskInfo(pMonitor, &diskInfo); + } + + taosArrayDestroy(clusterInfo.dnodes); + taosArrayDestroy(clusterInfo.mnodes); + taosArrayDestroy(vgroupInfo.vgroups); + taosArrayDestroy(diskInfo.datadirs); + + monSendReport(pMonitor); + monCleanupMonitorInfo(pMonitor); +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..37ff4ebc0540058790ff720b5f2b5d1830aff71e --- /dev/null +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); + dndReleaseWrapper(pWrapper); + } +} + +static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + if (msgFp == NULL) { + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + } + + return msgFp; +} + +static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { + SRpcConnInfo connInfo = {0}; + if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle); + return -1; + } + + memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); + memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); + + return 0; +} + +void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + + int32_t code = -1; + SNodeMsg *pMsg = NULL; + NodeMsgFp msgFp = NULL; + + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; + if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; + + dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); + if (pWrapper->procType == PROC_SINGLE) { + code = (*msgFp)(pWrapper->pMgmt, pMsg); + } else if (pWrapper->procType == PROC_PARENT) { + code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen); + } else { + } + +_OVER: + if (code == 0) { + if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + } else { + dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + dndReleaseWrapper(pWrapper); +} + +static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper != NULL) { + dndReleaseWrapper(pWrapper); + terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + dError("failed to create node since %s", terrstr()); + return -1; + } + + pWrapper = &pDnode->wrappers[ntype]; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been opened", pWrapper->name); + pWrapper->deployed = true; + } + + return code; +} + +static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + dError("failed to drop node since %s", terrstr()); + return -1; + } + + taosWLockLatch(&pWrapper->latch); + pWrapper->deployed = false; + + int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); + if (code != 0) { + pWrapper->deployed = true; + dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); + } else { + pWrapper->deployed = false; + dDebug("node:%s, has been dropped", pWrapper->name); + } + + taosWUnLockLatch(&pWrapper->latch); + dndReleaseWrapper(pWrapper); + return code; +} + +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { + switch (pMsg->rpcMsg.msgType) { + case TDMT_DND_CREATE_MNODE: + return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_DROP_MNODE: + return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_CREATE_QNODE: + return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_DROP_QNODE: + return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_CREATE_SNODE: + return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_DROP_SNODE: + return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_CREATE_BNODE: + return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); + case TDMT_DND_DROP_BNODE: + return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + return -1; + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndObj.c b/source/dnode/mgmt/container/src/dndObj.c new file mode 100644 index 0000000000000000000000000000000000000000..ff414b5f5e684739b97562e82b30b2981ff8f4c1 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndObj.c @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) { + pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes; + pDnode->serverPort = pOption->serverPort; + pDnode->dataDir = strdup(pOption->dataDir); + pDnode->localEp = strdup(pOption->localEp); + pDnode->localFqdn = strdup(pOption->localFqdn); + pDnode->firstEp = strdup(pOption->firstEp); + pDnode->secondEp = strdup(pOption->secondEp); + pDnode->pDisks = pOption->pDisks; + pDnode->numOfDisks = pOption->numOfDisks; + pDnode->rebootTime = taosGetTimestampMs(); + + if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL || + pDnode->secondEp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndClearMemory(SDnode *pDnode) { + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pMgmt = &pDnode->wrappers[n]; + taosMemoryFreeClear(pMgmt->path); + } + if (pDnode->pLockFile != NULL) { + taosUnLockFile(pDnode->pLockFile); + taosCloseFile(&pDnode->pLockFile); + pDnode->pLockFile = NULL; + } + taosMemoryFreeClear(pDnode->localEp); + taosMemoryFreeClear(pDnode->localFqdn); + taosMemoryFreeClear(pDnode->firstEp); + taosMemoryFreeClear(pDnode->secondEp); + taosMemoryFreeClear(pDnode->dataDir); + taosMemoryFree(pDnode); + dDebug("dnode object memory is cleared, data:%p", pDnode); +} + +SDnode *dndCreate(const SDnodeOpt *pOption) { + dInfo("start to create dnode object"); + int32_t code = -1; + char path[PATH_MAX] = {0}; + SDnode *pDnode = NULL; + + pDnode = taosMemoryCalloc(1, sizeof(SDnode)); + if (pDnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (dndInitMemory(pDnode, pOption) != 0) { + goto _OVER; + } + + dndSetStatus(pDnode, DND_STAT_INIT); + pDnode->pLockFile = dndCheckRunning(pDnode->dataDir); + if (pDnode->pLockFile == NULL) { + goto _OVER; + } + + if (dndInitServer(pDnode) != 0) { + dError("failed to init trans server since %s", terrstr()); + goto _OVER; + } + + if (dndInitClient(pDnode) != 0) { + dError("failed to init trans client since %s", terrstr()); + goto _OVER; + } + + dmGetMgmtFp(&pDnode->wrappers[DNODE]); + mmGetMgmtFp(&pDnode->wrappers[MNODE]); + vmGetMgmtFp(&pDnode->wrappers[VNODES]); + qmGetMgmtFp(&pDnode->wrappers[QNODE]); + smGetMgmtFp(&pDnode->wrappers[SNODE]); + bmGetMgmtFp(&pDnode->wrappers[BNODE]); + + if (dndInitMsgHandle(pDnode) != 0) { + goto _OVER; + } + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name); + pWrapper->path = strdup(path); + pWrapper->pDnode = pDnode; + if (pWrapper->path == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + pWrapper->procType = PROC_SINGLE; + taosInitRWLatch(&pWrapper->latch); + } + + code = 0; + +_OVER: + if (code != 0 && pDnode) { + dndClearMemory(pDnode); + pDnode = NULL; + dError("failed to create dnode object since %s", terrstr()); + } else { + dInfo("dnode object is created, data:%p", pDnode); + } + + return pDnode; +} + +void dndClose(SDnode *pDnode) { + if (pDnode == NULL) return; + + if (dndGetStatus(pDnode) == DND_STAT_STOPPED) { + dError("dnode is shutting down, data:%p", pDnode); + return; + } + + dInfo("start to close dnode, data:%p", pDnode); + dndSetStatus(pDnode, DND_STAT_STOPPED); + + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + dndCloseNode(pWrapper); + } + + dndClearMemory(pDnode); + dInfo("dnode object is closed, data:%p", pDnode); +} + +void dndHandleEvent(SDnode *pDnode, EDndEvent event) { + dInfo("dnode object receive event %d, data:%p", event, pDnode); + pDnode->event = event; +} + +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + SMgmtWrapper *pRetWrapper = pWrapper; + + taosRLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); + dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount); + } else { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + pRetWrapper = NULL; + } + taosRUnLockLatch(&pWrapper->latch); + + return pRetWrapper; +} + +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) { + int32_t code = 0; + + taosRLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); + dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount); + } else { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + code = -1; + } + taosRUnLockLatch(&pWrapper->latch); + + return code; +} + +void dndReleaseWrapper(SMgmtWrapper *pWrapper) { + if (pWrapper == NULL) return; + + taosRLockLatch(&pWrapper->latch); + int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); + taosRUnLockLatch(&pWrapper->latch); + dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount); +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c new file mode 100644 index 0000000000000000000000000000000000000000..4acb1f459ef91cefc2b6e7cd7c7a3200ac663d85 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" +#define INTERNAL_SECRET "_pwd" + +static inline void dndProcessQMVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) { + SMsgHead *pHead = pMsg->pCont; + int32_t vgId = htonl(pHead->vgId); + + SMgmtWrapper *pWrapper = pHandle->pWrapper; + if (vgId == QND_VGID) { + pWrapper = pHandle->pQndWrapper; + } else if (vgId == MND_VGID) { + pWrapper = pHandle->pMndWrapper; + } + + dTrace("msg:%s will be processed by %s, handle:%p app:%p vgId:%d", TMSG_INFO(pMsg->msgType), pWrapper->name, + pMsg->handle, pMsg->ahandle, vgId); + dndProcessRpcMsg(pWrapper, pMsg, pEpSet); +} + +static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { + SDnode *pDnode = parent; + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pRsp->msgType; + + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); + rpcFreeCont(pRsp->pCont); + return; + } + + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + if (pHandle->pWrapper != NULL) { + if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { + dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), + pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); + dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); + } else { + dndProcessQMVnodeRpcMsg(pHandle, pRsp, pEpSet); + } + } else { + dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); + rpcFreeCont(pRsp->pCont); + } +} + +int32_t dndInitClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = dndProcessResponse; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = INTERNAL_USER; + rpcInit.ckey = INTERNAL_CKEY; + rpcInit.spi = 1; + rpcInit.parent = pDnode; + + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + rpcInit.secret = pass; + + pMgmt->clientRpc = rpcOpen(&rpcInit); + if (pMgmt->clientRpc == NULL) { + dError("failed to init dnode rpc client"); + return -1; + } + + dDebug("dnode rpc client is initialized"); + return 0; +} + +void dndCleanupClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->clientRpc) { + rpcClose(pMgmt->clientRpc); + pMgmt->clientRpc = NULL; + dDebug("dnode rpc client is closed"); + } +} + +static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { + SDnode *pDnode = param; + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pReq->msgType; + + if (msgType == TDMT_DND_NETWORK_TEST) { + dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle); + dndProcessStartupReq(pDnode, pReq); + return; + } + + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pReq->pCont); + return; + } + + if (pReq->pCont == NULL) { + dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + return; + } + + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + if (pHandle->pWrapper != NULL) { + if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { + dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, + pReq->handle, pReq->ahandle); + dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); + } else { + dndProcessQMVnodeRpcMsg(pHandle, pReq, pEpSet); + } + } else { + dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pReq->pCont); + } +} + +static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { + STransMgmt *pMgmt = &pDnode->trans; + SEpSet epSet = {0}; + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + dndReleaseWrapper(pWrapper); + } + + rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); +} + +static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + int32_t code = 0; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + + if (strcmp(user, INTERNAL_USER) == 0) { + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { + taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); + } else { + code = -1; + } + + if (code == 0) { + memcpy(secret, pass, TSDB_PASSWORD_LEN); + *spi = 1; + *encrypt = 0; + *ckey = 0; + } + + return code; +} + +static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SDnode *pDnode = parent; + + if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) { + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + if (mmGetUserAuth(pWrapper, user, spi, encrypt, secret, ckey) == 0) { + dndReleaseWrapper(pWrapper); + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + dndReleaseWrapper(pWrapper); + } + + if (terrno != TSDB_CODE_APP_NOT_READY) { + dTrace("failed to get user auth from mnode since %s", terrstr()); + return -1; + } + + SAuthReq authReq = {0}; + tstrncpy(authReq.user, user, TSDB_USER_LEN); + int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSAuthReq(pReq, contLen, &authReq); + + SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; + SRpcMsg rpcRsp = {0}; + dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); + dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + terrno = rpcRsp.code; + dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); + } else { + SAuthRsp authRsp = {0}; + tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); + memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); + memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); + *spi = authRsp.spi; + *encrypt = authRsp.encrypt; + dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, + authRsp.encrypt); + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; +} + +int32_t dndInitServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); + if (numOfThreads < 1) { + numOfThreads = 1; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = pDnode->serverPort; + rpcInit.label = "DND"; + rpcInit.numOfThreads = numOfThreads; + rpcInit.cfp = dndProcessRequest; + rpcInit.sessions = tsMaxShellConns; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.afp = dndRetrieveUserAuthInfo; + rpcInit.parent = pDnode; + + pMgmt->serverRpc = rpcOpen(&rpcInit); + if (pMgmt->serverRpc == NULL) { + dError("failed to init dnode rpc server"); + return -1; + } + + dDebug("dnode rpc server is initialized"); + return 0; +} + +void dndCleanupServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->serverRpc) { + rpcClose(pMgmt->serverRpc); + pMgmt->serverRpc = NULL; + dDebug("dnode rpc server is closed"); + } +} + +int32_t dndInitMsgHandle(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + + for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { + NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; + int32_t vgId = pWrapper->msgVgIds[msgIndex]; + if (msgFp == NULL) continue; + + // dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); + + SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; + if (vgId == QND_VGID) { + if (pHandle->pQndWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pQndWrapper = pWrapper; + } else if (vgId == MND_VGID) { + if (pHandle->pMndWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pMndWrapper = pWrapper; + } else { + if (pHandle->pWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pWrapper = pWrapper; + } + } + } + + return 0; +} + +static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) { + if (pMgmt->clientRpc == NULL) { + terrno = TSDB_CODE_DND_OFFLINE; + return -1; + } + + rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); + return 0; +} + +int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq) { + if (pWrapper->procType == PROC_CHILD) { + } else { + SDnode *pDnode = pWrapper->pDnode; + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + terrno = TSDB_CODE_DND_OFFLINE; + dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); + return -1; + } + return dndSendRpcReq(&pDnode->trans, pEpSet, pReq); + } +} + +int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { + if (pWrapper->procType == PROC_CHILD) { + } else { + SDnode *pDnode = pWrapper->pDnode; + STransMgmt *pTrans = &pDnode->trans; + SEpSet epSet = {0}; + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + dndReleaseWrapper(pWrapper); + } + return dndSendRpcReq(pTrans, &epSet, pReq); + } +} + +void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { + if (pRsp->code == TSDB_CODE_APP_NOT_READY) { + SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); + if (pDnodeWrapper != NULL) { + dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); + dndReleaseWrapper(pDnodeWrapper); + } else { + rpcSendResponse(pRsp); + } + } else { + rpcSendResponse(pRsp); + } +} + +void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { + if (pWrapper->procType == PROC_CHILD) { + int32_t code = -1; + do { + code = taosProcPutToParentQueue(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen); + if (code != 0) { + taosMsleep(10); + } + } while (code != 0); + } else { + dndSendRpcRsp(pWrapper, pRsp); + } +} diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c deleted file mode 100644 index afb9ffbdd0b43cf22e8fdcacf4eabb8156faf2d3..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmnInt.h" -#include "tconfig.h" - -SDnodeObjCfg dmnGetObjCfg() { - SConfig *pCfg = taosGetCfg(); - SDnodeObjCfg objCfg = {0}; - - objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; - tstrncpy(objCfg.dataDir, tsDataDir, sizeof(objCfg.dataDir)); - tstrncpy(objCfg.firstEp, tsFirst, sizeof(objCfg.firstEp)); - tstrncpy(objCfg.secondEp, tsSecond, sizeof(objCfg.firstEp)); - objCfg.serverPort = tsServerPort; - tstrncpy(objCfg.localFqdn, tsLocalFqdn, sizeof(objCfg.localFqdn)); - snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); - objCfg.pDisks = tsDiskCfg; - objCfg.numOfDisks = tsDiskCfgNum; - return objCfg; -} - -void dmnDumpCfg() { - SConfig *pCfg = taosGetCfg(); - cfgDumpCfg(pCfg, 0, 1); -} \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c deleted file mode 100644 index 3d4de18dcb20ddb47faf7cd6dda1c7f8557166cb..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/daemon/src/dmnMain.c +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmnInt.h" - -static struct { - bool stop; - bool dumpConfig; - bool generateGrant; - bool printAuth; - bool printVersion; - char envFile[PATH_MAX]; - char apolloUrl[PATH_MAX]; -} dmn = {0}; - -static void dmnSigintHandle(int signum, void *info, void *ctx) { - uInfo("signal:%d is received", signum); - dmn.stop = true; -} - -static void dmnSetSignalHandle() { - taosSetSignal(SIGTERM, dmnSigintHandle); - taosSetSignal(SIGHUP, dmnSigintHandle); - taosSetSignal(SIGINT, dmnSigintHandle); - taosSetSignal(SIGABRT, dmnSigintHandle); - taosSetSignal(SIGBREAK, dmnSigintHandle); -} - -static void dmnWaitSignal() { - dmnSetSignalHandle(); - while (!dmn.stop) { - taosMsleep(100); - } -} - -static int32_t dmnParseOption(int32_t argc, char const *argv[]) { - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("config file path overflow"); - return -1; - } - tstrncpy(configDir, argv[i], PATH_MAX); - } else { - printf("'-c' requires a parameter, default is %s\n", configDir); - return -1; - } - } else if (strcmp(argv[i], "-C") == 0) { - dmn.dumpConfig = true; - } else if (strcmp(argv[i], "-k") == 0) { - dmn.generateGrant = true; - } else if (strcmp(argv[i], "-V") == 0) { - dmn.printVersion = true; - } else { - } - } - - return 0; -} - -int32_t dmnRunDnode() { - if (dndInit() != 0) { - uInfo("Failed to start TDengine, please check the log"); - return -1; - } - - SDnodeObjCfg objCfg = dmnGetObjCfg(); - SDnode *pDnode = dndCreate(&objCfg); - if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log"); - return -1; - } - - uInfo("Started TDengine service successfully."); - dmnWaitSignal(); - uInfo("TDengine is shut down!"); - - dndClose(pDnode); - dndCleanup(); - taosCloseLog(); - taosCleanupCfg(); - return 0; -} - -int main(int argc, char const *argv[]) { - if (!taosCheckSystemIsSmallEnd()) { - uError("TDengine does not run on non-small-end machines."); - return -1; - } - - if (dmnParseOption(argc, argv) != 0) { - return -1; - } - - if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; - } - - if (dmn.printVersion) { - dmnPrintVersion(); - return 0; - } - - if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { - uInfo("Failed to start TDengine since read config error"); - return -1; - } - - if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { - uInfo("Failed to start TDengine since read config error"); - return -1; - } - - if (dmn.dumpConfig) { - dmnDumpCfg(); - taosCleanupCfg(); - return 0; - } - - return dmnRunDnode(); -} diff --git a/source/dnode/mgmt/dnode/inc/dm.h b/source/dnode/mgmt/dnode/inc/dm.h new file mode 100644 index 0000000000000000000000000000000000000000..6c18d7969cab87627a0defd4b51e6ebf612b57d8 --- /dev/null +++ b/source/dnode/mgmt/dnode/inc/dm.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_DNODE_H_ +#define _TD_DND_DNODE_H_ + +#include "dnd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SDnodeMgmt SDnodeMgmt; + +void dmGetMgmtFp(SMgmtWrapper *pWrapper); +void dmInitMsgHandles(SMgmtWrapper *pWrapper); + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dnode/inc/dmInt.h b/source/dnode/mgmt/dnode/inc/dmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..b02b1d2297c6481702db7107b736bc83f56e6c32 --- /dev/null +++ b/source/dnode/mgmt/dnode/inc/dmInt.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_DNODE_INT_H_ +#define _TD_DND_DNODE_INT_H_ + +#include "dm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SDnodeMgmt { + int64_t dver; + int64_t updateTime; + int8_t statusSent; + SEpSet mnodeEpSet; + SHashObj *dnodeHash; + SArray *dnodeEps; + TdThread *threadId; + SRWLatch latch; + SSingleWorker mgmtWorker; + SSingleWorker statusWorker; + const char *path; + SDnode *pDnode; + SMgmtWrapper *pWrapper; +} SDnodeMgmt; + +// dmFile.c +int32_t dmReadFile(SDnodeMgmt *pMgmt); +int32_t dmWriteFile(SDnodeMgmt *pMgmt); +void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); + +// dmMsg.c +void dmSendStatusReq(SDnodeMgmt *pMgmt); +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); + +// dmWorker.c +int32_t dmStartWorker(SDnodeMgmt *pMgmt); +void dmStopWorker(SDnodeMgmt *pMgmt); +int32_t dmStartThread(SDnodeMgmt *pMgmt); +int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_DNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dnode/src/dmFile.c b/source/dnode/mgmt/dnode/src/dmFile.c new file mode 100644 index 0000000000000000000000000000000000000000..d44b1222a32f71a1273fb147166a59182aa65150 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmFile.c @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +static void dmPrintDnodes(SDnodeMgmt *pMgmt); +static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep); +static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps); + +int32_t dmReadFile(SDnodeMgmt *pMgmt) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 256 * 1024; + char *content = taosMemoryCalloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + SDnode *pDnode = pMgmt->pDnode; + + pMgmt->dnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); + if (pMgmt->dnodeEps == NULL) { + dError("failed to calloc dnodeEp array since %s", strerror(errno)); + goto PRASE_DNODE_OVER; + } + + snprintf(file, sizeof(file), "%s%sdnode.json", pMgmt->path, TD_DIRSEP); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_DNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_DNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_DNODE_OVER; + } + + cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s since dnodeId not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->dnodeId = dnodeId->valueint; + + cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); + if (!clusterId || clusterId->type != cJSON_String) { + dError("failed to read %s since clusterId not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->clusterId = atoll(clusterId->valuestring); + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->dropped = dropped->valueint; + + cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); + if (!dnodes || dnodes->type != cJSON_Array) { + dError("failed to read %s since dnodes not found", file); + goto PRASE_DNODE_OVER; + } + + int32_t numOfDnodes = cJSON_GetArraySize(dnodes); + if (numOfDnodes <= 0) { + dError("failed to read %s since numOfDnodes:%d invalid", file, numOfDnodes); + goto PRASE_DNODE_OVER; + } + + for (int32_t i = 0; i < numOfDnodes; ++i) { + cJSON *node = cJSON_GetArrayItem(dnodes, i); + if (node == NULL) break; + + SDnodeEp dnodeEp = {0}; + + cJSON *did = cJSON_GetObjectItem(node, "id"); + if (!did || did->type != cJSON_Number) { + dError("failed to read %s since dnodeId not found", file); + goto PRASE_DNODE_OVER; + } + + dnodeEp.id = dnodeId->valueint; + + cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); + if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { + dError("failed to read %s since dnodeFqdn not found", file); + goto PRASE_DNODE_OVER; + } + tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); + if (!dnodePort || dnodePort->type != cJSON_Number) { + dError("failed to read %s since dnodePort not found", file); + goto PRASE_DNODE_OVER; + } + + dnodeEp.ep.port = dnodePort->valueint; + + cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); + if (!isMnode || isMnode->type != cJSON_Number) { + dError("failed to read %s since isMnode not found", file); + goto PRASE_DNODE_OVER; + } + dnodeEp.isMnode = isMnode->valueint; + + taosArrayPush(pMgmt->dnodeEps, &dnodeEp); + } + + code = 0; + dInfo("succcessed to read file %s", file); + dmPrintDnodes(pMgmt); + +PRASE_DNODE_OVER: + if (content != NULL) taosMemoryFree(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + if (dmIsEpChanged(pMgmt, pDnode->dnodeId, pDnode->localEp)) { + dError("localEp %s different with %s and need reconfigured", pDnode->localEp, file); + return -1; + } + + if (taosArrayGetSize(pMgmt->dnodeEps) == 0) { + SDnodeEp dnodeEp = {0}; + dnodeEp.isMnode = 1; + taosGetFqdnPortFromEp(pDnode->firstEp, &dnodeEp.ep); + taosArrayPush(pMgmt->dnodeEps, &dnodeEp); + } + + dmResetDnodes(pMgmt, pMgmt->dnodeEps); + + terrno = code; + return code; +} + +int32_t dmWriteFile(SDnodeMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%sdnode.json.bak", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + dError("failed to write %s since %s", file, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 256 * 1024; + char *content = taosMemoryCalloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pDnode->dnodeId); + len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pDnode->clusterId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pDnode->dropped); + len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); + + int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + for (int32_t i = 0; i < numOfEps; ++i) { + SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->dnodeEps, i); + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); + len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); + if (i < numOfEps - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + taosMemoryFree(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + pMgmt->updateTime = taosGetTimestampMs(); + dDebug("successed to write %s", realfile); + return 0; +} + +void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *dnodeEps) { + int32_t numOfEps = taosArrayGetSize(dnodeEps); + if (numOfEps <= 0) return; + + taosWLockLatch(&pMgmt->latch); + + int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + if (numOfEps != numOfEpsOld) { + dmResetDnodes(pMgmt, dnodeEps); + dmWriteFile(pMgmt); + } else { + int32_t size = numOfEps * sizeof(SDnodeEp); + if (memcmp(pMgmt->dnodeEps->pData, dnodeEps->pData, size) != 0) { + dmResetDnodes(pMgmt, dnodeEps); + dmWriteFile(pMgmt); + } + } + + taosWUnLockLatch(&pMgmt->latch); +} + +static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps) { + if (pMgmt->dnodeEps != dnodeEps) { + SArray *tmp = pMgmt->dnodeEps; + pMgmt->dnodeEps = taosArrayDup(dnodeEps); + taosArrayDestroy(tmp); + } + + pMgmt->mnodeEpSet.inUse = 0; + pMgmt->mnodeEpSet.numOfEps = 0; + + int32_t mIndex = 0; + int32_t numOfEps = (int32_t)taosArrayGetSize(dnodeEps); + + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); + if (!pDnodeEp->isMnode) continue; + if (mIndex >= TSDB_MAX_REPLICA) continue; + pMgmt->mnodeEpSet.numOfEps++; + + pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep; + mIndex++; + } + + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); + taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); + } + + dmPrintDnodes(pMgmt); +} + +static void dmPrintDnodes(SDnodeMgmt *pMgmt) { + int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + dDebug("print dnode ep list, num:%d", numOfEps); + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pEp = taosArrayGet(pMgmt->dnodeEps, i); + dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); + } +} + +static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep) { + bool changed = false; + taosRLockLatch(&pMgmt->latch); + + SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); + if (pDnodeEp != NULL) { + char epstr[TSDB_EP_LEN + 1]; + snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); + changed = strcmp(ep, epstr) != 0; + } + + taosRUnLockLatch(&pMgmt->latch); + return changed; +} diff --git a/source/dnode/mgmt/dnode/src/dmInt.c b/source/dnode/mgmt/dnode/src/dmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..53049f7e7812bafc63e538b343250bb28371c560 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmInt.c @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) { + taosRLockLatch(&pMgmt->latch); + *pEpSet = pMgmt->mnodeEpSet; + taosRUnLockLatch(&pMgmt->latch); +} + +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) { + dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); + + taosWLockLatch(&pMgmt->latch); + pMgmt->mnodeEpSet = *pEpSet; + for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { + dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } + + taosWUnLockLatch(&pMgmt->latch); +} + +void dmGetDnodeEp(SMgmtWrapper *pWrapper, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + taosRLockLatch(&pMgmt->latch); + + SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); + if (pDnodeEp != NULL) { + if (pPort != NULL) { + *pPort = pDnodeEp->ep.port; + } + if (pFqdn != NULL) { + tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); + } + if (pEp != NULL) { + snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); + } + } + + taosRUnLockLatch(&pMgmt->latch); +} + +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) { + SDnode *pDnode = pMgmt->pDnode; + + SEpSet epSet = {0}; + dmGetMnodeEpSet(pMgmt, &epSet); + + dDebug("RPC %p, req is redirected, num:%d use:%d", pReq->handle, epSet.numOfEps, epSet.inUse); + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + if (strcmp(epSet.eps[i].fqdn, pDnode->localFqdn) == 0 && epSet.eps[i].port == pDnode->serverPort) { + epSet.inUse = (i + 1) % epSet.numOfEps; + } + + epSet.eps[i].port = htons(epSet.eps[i].port); + } + + rpcSendRedirectRsp(pReq->handle, &epSet); +} + +static int32_t dmStart(SMgmtWrapper *pWrapper) { + dDebug("dnode-mgmt start to run"); + return dmStartThread(pWrapper->pMgmt); +} + +int32_t dmInit(SMgmtWrapper *pWrapper) { + SDnode *pDnode = pWrapper->pDnode; + SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); + dInfo("dnode-mgmt start to init"); + + pDnode->dnodeId = 0; + pDnode->dropped = 0; + pDnode->clusterId = 0; + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pDnode; + pMgmt->pWrapper = pWrapper; + taosInitRWLatch(&pMgmt->latch); + + pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMgmt->dnodeHash == NULL) { + dError("failed to init dnode hash"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (dmReadFile(pMgmt) != 0) { + dError("failed to read file since %s", terrstr()); + return -1; + } + + if (pDnode->dropped) { + dError("dnode will not start since its already dropped"); + return -1; + } + + if (dmStartWorker(pMgmt) != 0) { + return -1; + } + + pWrapper->pMgmt = pMgmt; + dInfo("dnode-mgmt is initialized"); + return 0; +} + +void dmCleanup(SMgmtWrapper *pWrapper) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("dnode-mgmt start to clean up"); + dmStopWorker(pMgmt); + + taosWLockLatch(&pMgmt->latch); + + if (pMgmt->dnodeEps != NULL) { + taosArrayDestroy(pMgmt->dnodeEps); + pMgmt->dnodeEps = NULL; + } + + if (pMgmt->dnodeHash != NULL) { + taosHashCleanup(pMgmt->dnodeHash); + pMgmt->dnodeHash = NULL; + } + + taosWUnLockLatch(&pMgmt->latch); + + taosMemoryFree(pMgmt); + pWrapper->pMgmt = NULL; + dInfo("dnode-mgmt is cleaned up"); +} + +int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { + *required = true; + return 0; +} + +void dmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = dmInit; + mgmtFp.closeFp = dmCleanup; + mgmtFp.startFp = dmStart; + mgmtFp.requiredFp = dmRequire; + + dmInitMsgHandles(pWrapper); + pWrapper->name = "dnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/dnode/src/dmMsg.c b/source/dnode/mgmt/dnode/src/dmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..836817e772a4a64fd541ca7e565d28b198143c9f --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmMsg.c @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" +#include "vm.h" + +void dmSendStatusReq(SDnodeMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + SStatusReq req = {0}; + + taosRLockLatch(&pMgmt->latch); + req.sver = tsVersion; + req.dver = pMgmt->dver; + req.dnodeId = pDnode->dnodeId; + req.clusterId = pDnode->clusterId; + req.rebootTime = pDnode->rebootTime; + req.updateTime = pMgmt->updateTime; + req.numOfCores = tsNumOfCores; + req.numOfSupportVnodes = pDnode->numOfSupportVnodes; + tstrncpy(req.dnodeEp, pDnode->localEp, TSDB_EP_LEN); + + req.clusterCfg.statusInterval = tsStatusInterval; + req.clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); + taosRUnLockLatch(&pMgmt->latch); + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); + vmMonitorVnodeLoads(pWrapper, req.pVloads); + dndReleaseWrapper(pWrapper); + } + + int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); + void *pHead = rpcMallocCont(contLen); + tSerializeSStatusReq(pHead, contLen, &req); + taosArrayDestroy(req.pVloads); + + SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; + pMgmt->statusSent = 1; + + dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); + dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg); +} + +static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { + SDnode *pDnode = pMgmt->pDnode; + + if (pDnode->dnodeId == 0) { + dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); + taosWLockLatch(&pMgmt->latch); + pDnode->dnodeId = pCfg->dnodeId; + pDnode->clusterId = pCfg->clusterId; + dmWriteFile(pMgmt); + taosWUnLockLatch(&pMgmt->latch); + } +} + +int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pRsp = &pMsg->rpcMsg; + + if (pRsp->code != TSDB_CODE_SUCCESS) { + if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pDnode->dropped && pDnode->dnodeId > 0) { + dInfo("dnode:%d, set to dropped since not exist in mnode", pDnode->dnodeId); + pDnode->dropped = 1; + dmWriteFile(pMgmt); + } + } else { + SStatusRsp statusRsp = {0}; + if (pRsp->pCont != NULL && pRsp->contLen != 0 && + tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { + pMgmt->dver = statusRsp.dver; + dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); + dmUpdateDnodeEps(pMgmt, statusRsp.pDnodeEps); + } + tFreeSStatusRsp(&statusRsp); + } + + pMgmt->statusSent = 0; +} + +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("auth rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("grant rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SDCfgDnodeReq *pCfg = pReq->pCont; + dError("config req is received, but not supported yet"); + return TSDB_CODE_OPS_NOT_SUPPORT; +} + +void dmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by DNODE + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + + // Requests handled by MNODE + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); +} diff --git a/source/dnode/mgmt/dnode/src/dmWorker.c b/source/dnode/mgmt/dnode/src/dmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..d34a26436c2184dfd930b66615ee56e2e34cd5c0 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmWorker.c @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bm.h" +#include "dmInt.h" +#include "mm.h" +#include "qm.h" +#include "sm.h" +#include "vm.h" + +static void *dmThreadRoutine(void *param) { + SDnodeMgmt *pMgmt = param; + SDnode * pDnode = pMgmt->pDnode; + int64_t lastStatusTime = taosGetTimestampMs(); + int64_t lastMonitorTime = lastStatusTime; + + setThreadName("dnode-hb"); + + while (true) { + taosThreadTestCancel(); + taosMsleep(200); + if (dndGetStatus(pDnode) != DND_STAT_RUNNING || pDnode->dropped) { + continue; + } + + int64_t curTime = taosGetTimestampMs(); + + float statusInterval = (curTime - lastStatusTime) / 1000.0f; + if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { + dmSendStatusReq(pMgmt); + lastStatusTime = curTime; + } + + float monitorInterval = (curTime - lastMonitorTime) / 1000.0f; + if (monitorInterval >= tsMonitorInterval) { + dndSendMonitorReport(pDnode); + lastMonitorTime = curTime; + } + } +} + +static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SDnodeMgmt *pMgmt = pInfo->ahandle; + + SDnode * pDnode = pMgmt->pDnode; + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + dTrace("msg:%p, will be processed in dnode queue", pMsg); + + switch (pRpc->msgType) { + case TDMT_DND_CREATE_MNODE: + case TDMT_DND_CREATE_QNODE: + case TDMT_DND_CREATE_SNODE: + case TDMT_DND_CREATE_BNODE: + case TDMT_DND_DROP_MNODE: + case TDMT_DND_DROP_QNODE: + case TDMT_DND_DROP_SNODE: + case TDMT_DND_DROP_BNODE: + code = dndProcessNodeMsg(pMgmt->pDnode, pMsg); + break; + case TDMT_DND_CONFIG_DNODE: + code = dmProcessConfigReq(pMgmt, pMsg); + break; + case TDMT_MND_STATUS_RSP: + code = dmProcessStatusRsp(pMgmt, pMsg); + break; + case TDMT_MND_AUTH_RSP: + code = dmProcessAuthRsp(pMgmt, pMsg); + break; + case TDMT_MND_GRANT_RSP: + code = dmProcessGrantRsp(pMgmt, pMsg); + break; + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType)); + } + + if (pRpc->msgType & 1u) { + if (code != 0) code = terrno; + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; + rpcSendResponse(&rsp); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +int32_t dmStartWorker(SDnodeMgmt *pMgmt) { + SSingleWorkerCfg mgmtCfg = { + .minNum = 1, .maxNum = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); + return -1; + } + + SSingleWorkerCfg statusCfg = { + .minNum = 1, .maxNum = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) { + dError("failed to start dnode status worker since %s", terrstr()); + return -1; + } + + dDebug("dnode workers are initialized"); + return 0; +} + +int32_t dmStartThread(SDnodeMgmt *pMgmt) { + pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); + if (pMgmt->threadId == NULL) { + dError("failed to init dnode thread"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +void dmStopWorker(SDnodeMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->mgmtWorker); + tSingleWorkerCleanup(&pMgmt->statusWorker); + + if (pMgmt->threadId != NULL) { + taosDestoryThread(pMgmt->threadId); + pMgmt->threadId = NULL; + } + dDebug("dnode workers are closed"); +} + +int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SSingleWorker *pWorker = &pMgmt->mgmtWorker; + if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { + pWorker = &pMgmt->statusWorker; + } + + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + return taosWriteQitem(pWorker->queue, pMsg); +} diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt deleted file mode 100644 index c99a4055274f274f87cb030fa07a20263d67f88d..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -aux_source_directory(src DNODE_SRC) -add_library(dnode STATIC ${DNODE_SRC}) -target_link_libraries( - dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor -) -target_include_directories( - dnode - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h deleted file mode 100644 index aeea5386b4ea39d42cac8d599e53d595f71c5660..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_ENV_H_ -#define _TD_DND_ENV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "dndInt.h" - -typedef struct { - EWorkerType type; - const char *name; - int32_t minNum; - int32_t maxNum; - void *queueFp; - SDnode *pDnode; - STaosQueue *queue; - union { - SQWorkerPool pool; - SWWorkerPool mpool; - }; -} SDnodeWorker; - -typedef struct { - char *dnode; - char *mnode; - char *snode; - char *bnode; - char *vnodes; -} SDnodeDir; - -typedef struct { - int32_t dnodeId; - int32_t dropped; - int64_t clusterId; - int64_t dver; - int64_t rebootTime; - int64_t updateTime; - int8_t statusSent; - SEpSet mnodeEpSet; - char *file; - SHashObj *dnodeHash; - SArray *pDnodeEps; - pthread_t *threadId; - SRWLatch latch; - SDnodeWorker mgmtWorker; - SDnodeWorker statusWorker; -} SDnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SMnode *pMnode; - SRWLatch latch; - SDnodeWorker readWorker; - SDnodeWorker writeWorker; - SDnodeWorker syncWorker; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; -} SMnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SQnode *pQnode; - SRWLatch latch; - SDnodeWorker queryWorker; - SDnodeWorker fetchWorker; -} SQnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - int8_t uniqueWorkerInUse; - SSnode *pSnode; - SRWLatch latch; - SArray *uniqueWorkers; // SArray - SDnodeWorker sharedWorker; -} SSnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SBnode *pBnode; - SRWLatch latch; - SDnodeWorker writeWorker; -} SBnodeMgmt; - -typedef struct { - int32_t openVnodes; - int32_t totalVnodes; - int32_t masterNum; - int64_t numOfSelectReqs; - int64_t numOfInsertReqs; - int64_t numOfInsertSuccessReqs; - int64_t numOfBatchInsertReqs; - int64_t numOfBatchInsertSuccessReqs; -} SVnodesStat; - -typedef struct { - SVnodesStat stat; - SHashObj *hash; - SRWLatch latch; - SQWorkerPool queryPool; - SFWorkerPool fetchPool; - SWWorkerPool syncPool; - SWWorkerPool writePool; -} SVnodesMgmt; - -typedef struct { - void *serverRpc; - void *clientRpc; - DndMsgFp msgFp[TDMT_MAX]; -} STransMgmt; - -typedef struct SDnode { - EStat stat; - SDnodeObjCfg cfg; - SDnodeDir dir; - TdFilePtr pLockFile; - SDnodeMgmt dmgmt; - SMnodeMgmt mmgmt; - SQnodeMgmt qmgmt; - SSnodeMgmt smgmt; - SBnodeMgmt bmgmt; - SVnodesMgmt vmgmt; - STransMgmt tmgmt; - STfs *pTfs; - SStartupReq startup; -} SDnode; - -int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_ENV_H_*/ diff --git a/source/dnode/mgmt/impl/inc/dndMgmt.h b/source/dnode/mgmt/impl/inc/dndMgmt.h deleted file mode 100644 index 9cc0c4ae66da13c41c56a298e8a7839fa2a2095f..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndMgmt.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_DNODE_H_ -#define _TD_DND_DNODE_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitMgmt(SDnode *pDnode); -void dndStopMgmt(SDnode *pDnode); -void dndCleanupMgmt(SDnode *pDnode); - -int32_t dndGetDnodeId(SDnode *pDnode); -int64_t dndGetClusterId(SDnode *pDnode); -void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort); -void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet); - -void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg); -void dndSendStatusReq(SDnode *pDnode); -void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet); -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndMnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h deleted file mode 100644 index 0f03bb3832fb7c66bf651bdec18ec6ca196643d9..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndMnode.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_MNODE_H_ -#define _TD_DND_MNODE_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitMnode(SDnode *pDnode); -void dndCleanupMnode(SDnode *pDnode); - -int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); -void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); - -int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndSnode.h b/source/dnode/mgmt/impl/inc/dndSnode.h deleted file mode 100644 index f72d2a137a304d5e9058e84d140f077a15a42c9f..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndSnode.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_SNODE_H_ -#define _TD_DND_SNODE_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitSnode(SDnode *pDnode); -void dndCleanupSnode(SDnode *pDnode); - -// void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); - -void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h deleted file mode 100644 index 895e94060f741722830f5e71d06c749a2c74dec6..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_VNODES_H_ -#define _TD_DND_VNODES_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitVnodes(SDnode *pDnode); -void dndCleanupVnodes(SDnode *pDnode); -void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads); -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); - -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq); - -int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pReq); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c deleted file mode 100644 index 81b020c1523e8f3443dc1ee9e68c3e52716c4337..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ /dev/null @@ -1,392 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndBnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); - -static SBnode *dndAcquireBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - SBnode *pBnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pBnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pBnode = pMgmt->pBnode; - } else { - terrno = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pBnode != NULL) { - dTrace("acquire bnode, refCount:%d", refCount); - } - return pBnode; -} - -static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) { - if (pBnode == NULL) return; - - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release bnode, refCount:%d", refCount); -} - -static int32_t dndReadBnodeFile(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - int32_t code = TSDB_CODE_DND_BNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_BNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_BNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_BNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_BNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_BNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_BNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteBnodeFile(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartBnodeWorker(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, dndProcessBnodeQueue) != 0) { - dError("failed to start bnode write worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopBnodeWorker(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->writeWorker); -} - -static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode != NULL) { - dndReleaseBnode(pDnode, pBnode); - terrno = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; - dError("failed to create bnode since %s", terrstr()); - return -1; - } - - SBnodeOpt option = {0}; - dndBuildBnodeOption(pDnode, &option); - - pBnode = bndOpen(pDnode->dir.bnode, &option); - if (pBnode == NULL) { - dError("failed to open bnode since %s", terrstr()); - return -1; - } - - if (dndStartBnodeWorker(pDnode) != 0) { - dError("failed to start bnode worker since %s", terrstr()); - bndClose(pBnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteBnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write bnode file since %s", terrstr()); - dndStopBnodeWorker(pDnode); - bndClose(pBnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pBnode = pBnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("bnode open successfully"); - return 0; -} - -static int32_t dndDropBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode == NULL) { - dError("failed to drop bnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteBnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseBnode(pDnode, pBnode); - dError("failed to drop bnode since %s", terrstr()); - return -1; - } - - dndReleaseBnode(pDnode, pBnode); - dndStopBnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteBnodeFile(pDnode); - bndClose(pBnode); - pMgmt->pBnode = NULL; - bndDestroy(pDnode->dir.bnode); - - return 0; -} - -int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateBnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION; - dError("failed to create bnode since %s", terrstr()); - return -1; - } else { - return dndOpenBnode(pDnode); - } -} - -int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropBnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION; - dError("failed to drop bnode since %s", terrstr()); - return -1; - } else { - return dndDropBnode(pDnode); - } -} - -static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dndSendBnodeErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - dndSendBnodeErrorRsp(pMsg, code); - } -} - -static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode == NULL) { - dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); - return; - } - - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - if (pArray == NULL) { - dndReleaseBnode(pDnode, pBnode); - dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); - return; - } - - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - void *ptr = taosArrayPush(pArray, &pMsg); - if (ptr == NULL) { - dndSendBnodeErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); - } - } - - bndProcessWMsgs(pBnode, pArray); - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - taosArrayDestroy(pArray); - dndReleaseBnode(pDnode, pBnode); -} - -static void dndWriteBnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; - - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseBnode(pDnode, pBnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteBnodeMsgToWorker(pDnode, &pDnode->bmgmt.writeWorker, pMsg); -} - -int32_t dndInitBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadBnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("bnode has been deployed and needs to be deleted"); - bndDestroy(pDnode->dir.bnode); - return 0; - } - - if (!pMgmt->deployed) return 0; - - return dndOpenBnode(pDnode); -} - -void dndCleanupBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - if (pMgmt->pBnode) { - dndStopBnodeWorker(pDnode); - bndClose(pMgmt->pBnode); - pMgmt->pBnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c deleted file mode 100644 index 84b2dca3264ec6196ede30e99f6f9c08c2efcacc..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndBnode.h" -#include "dndMgmt.h" -#include "dndMnode.h" -#include "dndQnode.h" -#include "dndSnode.h" -#include "dndTransport.h" -#include "dndVnodes.h" -#include "monitor.h" -#include "sync.h" -#include "tfs.h" -#include "wal.h" - -static int8_t once = DND_ENV_INIT; - -EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } - -void dndSetStat(SDnode *pDnode, EStat stat) { - dDebug("dnode status set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat)); - pDnode->stat = stat; -} - -const char *dndStatStr(EStat stat) { - switch (stat) { - case DND_STAT_INIT: - return "init"; - case DND_STAT_RUNNING: - return "running"; - case DND_STAT_STOPPED: - return "stopped"; - default: - return "unknown"; - } -} - -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); -} - -static TdFilePtr dndCheckRunning(char *dataDir) { - char filepath[PATH_MAX] = {0}; - snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); - - TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - return NULL; - } - - int32_t ret = taosLockFile(pFile); - if (ret != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - taosCloseFile(&pFile); - return NULL; - } - - return pFile; -} - -static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) { - pDnode->pLockFile = dndCheckRunning(pCfg->dataDir); - if (pDnode->pLockFile == NULL) { - return -1; - } - - char path[PATH_MAX + 100]; - snprintf(path, sizeof(path), "%s%smnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.mnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%svnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.vnodes = tstrdup(path); - snprintf(path, sizeof(path), "%s%sdnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.dnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%ssnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.snode = tstrdup(path); - snprintf(path, sizeof(path), "%s%sbnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.bnode = tstrdup(path); - - if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL || - pDnode->dir.snode == NULL || pDnode->dir.bnode == NULL) { - dError("failed to malloc dir object"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (taosMkDir(pDnode->dir.dnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.dnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.mnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.mnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.vnodes) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.vnodes, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.snode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.snode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.bnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.bnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); - return 0; -} - -static void dndCloseDir(SDnode *pDnode) { - tfree(pDnode->dir.mnode); - tfree(pDnode->dir.vnodes); - tfree(pDnode->dir.dnode); - tfree(pDnode->dir.snode); - tfree(pDnode->dir.bnode); - - if (pDnode->pLockFile != NULL) { - taosUnLockFile(pDnode->pLockFile); - taosCloseFile(&pDnode->pLockFile); - pDnode->pLockFile = NULL; - } -} - -SDnode *dndCreate(SDnodeObjCfg *pCfg) { - dInfo("start to create dnode object"); - - SDnode *pDnode = calloc(1, sizeof(SDnode)); - if (pDnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - dError("failed to create dnode object since %s", terrstr()); - return NULL; - } - - dndSetStat(pDnode, DND_STAT_INIT); - - if (dndInitDir(pDnode, pCfg) != 0) { - dError("failed to init dnode dir since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - SDiskCfg dCfg = {0}; - tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN); - dCfg.level = 0; - dCfg.primary = 1; - SDiskCfg *pDisks = pDnode->cfg.pDisks; - int32_t numOfDisks = pDnode->cfg.numOfDisks; - if (numOfDisks <= 0 || pDisks == NULL) { - pDisks = &dCfg; - numOfDisks = 1; - } - - pDnode->pTfs = tfsOpen(pDisks, numOfDisks); - if (pDnode->pTfs == NULL) { - dError("failed to init tfs since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitMgmt(pDnode) != 0) { - dError("failed to init mgmt since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitVnodes(pDnode) != 0) { - dError("failed to init vnodes since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitQnode(pDnode) != 0) { - dError("failed to init qnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitSnode(pDnode) != 0) { - dError("failed to init snode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitBnode(pDnode) != 0) { - dError("failed to init bnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitMnode(pDnode) != 0) { - dError("failed to init mnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitTrans(pDnode) != 0) { - dError("failed to init transport since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - dndSetStat(pDnode, DND_STAT_RUNNING); - dndSendStatusReq(pDnode); - dndReportStartup(pDnode, "TDengine", "initialized successfully"); - dInfo("dnode object is created, data:%p", pDnode); - - return pDnode; -} - -void dndClose(SDnode *pDnode) { - if (pDnode == NULL) return; - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("dnode is shutting down, data:%p", pDnode); - return; - } - - dInfo("start to close dnode, data:%p", pDnode); - dndSetStat(pDnode, DND_STAT_STOPPED); - dndCleanupTrans(pDnode); - dndStopMgmt(pDnode); - dndCleanupMnode(pDnode); - dndCleanupBnode(pDnode); - dndCleanupSnode(pDnode); - dndCleanupQnode(pDnode); - dndCleanupVnodes(pDnode); - dndCleanupMgmt(pDnode); - tfsClose(pDnode->pTfs); - - dndCloseDir(pDnode); - free(pDnode); - dInfo("dnode object is closed, data:%p", pDnode); -} - -int32_t dndInit() { - if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { - terrno = TSDB_CODE_REPEAT_INIT; - dError("failed to init dnode env since %s", terrstr()); - return -1; - } - - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); - - if (rpcInit() != 0) { - dError("failed to init rpc since %s", terrstr()); - dndCleanup(); - return -1; - } - - if (walInit() != 0) { - dError("failed to init wal since %s", terrstr()); - dndCleanup(); - return -1; - } - - SVnodeOpt vnodeOpt = { - .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode}; - - if (vnodeInit(&vnodeOpt) != 0) { - dError("failed to init vnode since %s", terrstr()); - dndCleanup(); - return -1; - } - - SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn, .comp = tsMonitorComp}; - if (monInit(&monCfg) != 0) { - dError("failed to init monitor since %s", terrstr()); - dndCleanup(); - return -1; - } - - dInfo("dnode env is initialized"); - return 0; -} - -void dndCleanup() { - if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { - dError("dnode env is already cleaned up"); - return; - } - - walCleanUp(); - vnodeCleanup(); - rpcCleanup(); - monCleanup(); - - taosStopCacheRefreshWorker(); - dInfo("dnode env is cleaned up"); -} - -int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { - tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); - pInfo->logdir.size = tsLogSpace.size; - tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); - pInfo->tempdir.size = tsTempSpace.size; - - return tfsGetMonitorInfo(pDnode->pTfs, pInfo); -} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c deleted file mode 100644 index 866076a3d0c5b19710ff1c1b6e68ff8285107b31..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ /dev/null @@ -1,763 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndMgmt.h" -#include "dndBnode.h" -#include "dndMnode.h" -#include "dndQnode.h" -#include "dndSnode.h" -#include "dndTransport.h" -#include "dndVnodes.h" -#include "dndWorker.h" -#include "monitor.h" - -static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static int32_t dndReadDnodes(SDnode *pDnode); -static int32_t dndWriteDnodes(SDnode *pDnode); -static void *dnodeThreadRoutine(void *param); - -static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq); -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp); -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pRsp); -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pRsp); - -int32_t dndGetDnodeId(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t dnodeId = pMgmt->dnodeId; - taosRUnLockLatch(&pMgmt->latch); - return dnodeId; -} - -int64_t dndGetClusterId(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - int64_t clusterId = pMgmt->clusterId; - taosRUnLockLatch(&pMgmt->latch); - return clusterId; -} - -void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - - SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); - if (pDnodeEp != NULL) { - if (pPort != NULL) { - *pPort = pDnodeEp->ep.port; - } - if (pFqdn != NULL) { - tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); - } - if (pEp != NULL) { - snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); - } - } - - taosRUnLockLatch(&pMgmt->latch); -} - -void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - *pEpSet = pMgmt->mnodeEpSet; - taosRUnLockLatch(&pMgmt->latch); -} - -void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) { - tmsg_t msgType = pReq->msgType; - - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - - dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - if (strcmp(epSet.eps[i].fqdn, pDnode->cfg.localFqdn) == 0 && epSet.eps[i].port == pDnode->cfg.serverPort) { - epSet.inUse = (i + 1) % epSet.numOfEps; - } - - epSet.eps[i].port = htons(epSet.eps[i].port); - } - - rpcSendRedirectRsp(pReq->handle, &epSet); -} - -static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - pMgmt->mnodeEpSet = *pEpSet; - for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } - - taosWUnLockLatch(&pMgmt->latch); -} - -static void dndPrintDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - dDebug("print dnode ep list, num:%d", numOfEps); - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pEp = taosArrayGet(pMgmt->pDnodeEps, i); - dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); - } -} - -static void dndResetDnodes(SDnode *pDnode, SArray *pDnodeEps) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pMgmt->pDnodeEps != pDnodeEps) { - SArray *tmp = pMgmt->pDnodeEps; - pMgmt->pDnodeEps = taosArrayDup(pDnodeEps); - taosArrayDestroy(tmp); - } - - pMgmt->mnodeEpSet.inUse = 0; - pMgmt->mnodeEpSet.numOfEps = 0; - - int32_t mIndex = 0; - int32_t numOfEps = (int32_t)taosArrayGetSize(pDnodeEps); - - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i); - if (!pDnodeEp->isMnode) continue; - if (mIndex >= TSDB_MAX_REPLICA) continue; - pMgmt->mnodeEpSet.numOfEps++; - - pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep; - mIndex++; - } - - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i); - taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); - } - - dndPrintDnodes(pDnode); -} - -static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) { - bool changed = false; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - - SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); - if (pDnodeEp != NULL) { - char epstr[TSDB_EP_LEN + 1]; - snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); - changed = strcmp(pEp, epstr) != 0; - } - - taosRUnLockLatch(&pMgmt->latch); - return changed; -} - -static int32_t dndReadDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - pMgmt->pDnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); - if (pMgmt->pDnodeEps == NULL) { - dError("failed to calloc dnodeEp array since %s", strerror(errno)); - goto PRASE_DNODE_OVER; - } - - int32_t code = TSDB_CODE_DND_DNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - // fp = fopen(pMgmt->file, "r"); - TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", pMgmt->file); - code = 0; - goto PRASE_DNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->dnodeId = dnodeId->valueint; - - cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); - if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read %s since clusterId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->clusterId = atoll(clusterId->valuestring); - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); - if (!dnodes || dnodes->type != cJSON_Array) { - dError("failed to read %s since dnodes not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - int32_t numOfDnodes = cJSON_GetArraySize(dnodes); - if (numOfDnodes <= 0) { - dError("failed to read %s since numOfDnodes:%d invalid", pMgmt->file, numOfDnodes); - goto PRASE_DNODE_OVER; - } - - for (int32_t i = 0; i < numOfDnodes; ++i) { - cJSON *node = cJSON_GetArrayItem(dnodes, i); - if (node == NULL) break; - - SDnodeEp dnodeEp = {0}; - - cJSON *did = cJSON_GetObjectItem(node, "id"); - if (!did || did->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - dnodeEp.id = dnodeId->valueint; - - cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { - dError("failed to read %s since dnodeFqdn not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); - if (!dnodePort || dnodePort->type != cJSON_Number) { - dError("failed to read %s since dnodePort not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - dnodeEp.ep.port = dnodePort->valueint; - - cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); - if (!isMnode || isMnode->type != cJSON_Number) { - dError("failed to read %s since isMnode not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - dnodeEp.isMnode = isMnode->valueint; - - taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); - } - - code = 0; - dInfo("succcessed to read file %s", pMgmt->file); - dndPrintDnodes(pDnode); - -PRASE_DNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { - dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); - return -1; - } - - if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) { - SDnodeEp dnodeEp = {0}; - dnodeEp.isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep); - taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); - } - - dndResetDnodes(pDnode, pMgmt->pDnodeEps); - - terrno = 0; - return 0; -} - -static int32_t dndWriteDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - // FILE *fp = fopen(pMgmt->file, "w"); - TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - dError("failed to write %s since %s", pMgmt->file, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pMgmt->dnodeId); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pMgmt->clusterId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); - - int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - for (int32_t i = 0; i < numOfEps; ++i) { - SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->pDnodeEps, i); - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); - len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); - if (i < numOfEps - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - terrno = 0; - - pMgmt->updateTime = taosGetTimestampMs(); - dDebug("successed to write %s", pMgmt->file); - return 0; -} - -void dndSendStatusReq(SDnode *pDnode) { - SStatusReq req = {0}; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - req.sver = tsVersion; - req.dver = pMgmt->dver; - req.dnodeId = pMgmt->dnodeId; - req.clusterId = pMgmt->clusterId; - req.rebootTime = pMgmt->rebootTime; - req.updateTime = pMgmt->updateTime; - req.numOfCores = tsNumOfCores; - req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes; - memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); - - req.clusterCfg.statusInterval = tsStatusInterval; - req.clusterCfg.checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); - memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); - taosRUnLockLatch(&pMgmt->latch); - - req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); - dndGetVnodeLoads(pDnode, req.pVloads); - - int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); - tSerializeSStatusReq(pHead, contLen, &req); - taosArrayDestroy(req.pVloads); - - SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; - pMgmt->statusSent = 1; - - dTrace("pDnode:%p, send status req to mnode", pDnode); - dndSendReqToMnode(pDnode, &rpcMsg); -} - -static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - if (pMgmt->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); - taosWLockLatch(&pMgmt->latch); - pMgmt->dnodeId = pCfg->dnodeId; - pMgmt->clusterId = pCfg->clusterId; - dndWriteDnodes(pDnode); - taosWUnLockLatch(&pMgmt->latch); - } -} - -static void dndUpdateDnodeEps(SDnode *pDnode, SArray *pDnodeEps) { - int32_t numOfEps = taosArrayGetSize(pDnodeEps); - if (numOfEps <= 0) return; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - if (numOfEps != numOfEpsOld) { - dndResetDnodes(pDnode, pDnodeEps); - dndWriteDnodes(pDnode); - } else { - int32_t size = numOfEps * sizeof(SDnodeEp); - if (memcmp(pMgmt->pDnodeEps->pData, pDnodeEps->pData, size) != 0) { - dndResetDnodes(pDnode, pDnodeEps); - dndWriteDnodes(pDnode); - } - } - - taosWUnLockLatch(&pMgmt->latch); -} - -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pRsp->code != TSDB_CODE_SUCCESS) { - if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) { - dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->dnodeId); - pMgmt->dropped = 1; - dndWriteDnodes(pDnode); - } - } else { - SStatusRsp statusRsp = {0}; - if (pRsp->pCont != NULL && pRsp->contLen != 0 && - tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { - pMgmt->dver = statusRsp.dver; - dndUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg); - dndUpdateDnodeEps(pDnode, statusRsp.pDnodeEps); - } - taosArrayDestroy(statusRsp.pDnodeEps); - } - - pMgmt->statusSent = 0; -} - -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pReq) { dError("auth rsp is received, but not supported yet"); } - -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pReq) { - dError("grant rsp is received, but not supported yet"); -} - -static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - dError("config req is received, but not supported yet"); - SDCfgDnodeReq *pCfg = pReq->pCont; - return TSDB_CODE_OPS_NOT_SUPPORT; -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - - SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; - rpcSendResponse(&rpcRsp); -} - -static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { - pInfo->dnode_id = dndGetDnodeId(pDnode); - tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); - pInfo->cluster_id = dndGetClusterId(pDnode); - pInfo->protocol = 1; -} - -static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f); - taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); - pInfo->cpu_cores = tsNumOfCores; - taosGetProcMemory(&pInfo->mem_engine); - taosGetSysMemory(&pInfo->mem_system); - pInfo->mem_total = tsTotalMemoryKB; - pInfo->disk_engine = 0; - pInfo->disk_used = tsDataSpace.size.used; - pInfo->disk_total = tsDataSpace.size.total; - taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); - taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); - - SVnodesStat *pStat = &pDnode->vmgmt.stat; - pInfo->req_select = pStat->numOfSelectReqs; - pInfo->req_insert = pStat->numOfInsertReqs; - pInfo->req_insert_success = pStat->numOfInsertSuccessReqs; - pInfo->req_insert_batch = pStat->numOfBatchInsertReqs; - pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs; - pInfo->errors = tsNumOfErrorLogs; - pInfo->vnodes_num = pStat->totalVnodes; - pInfo->masters = pStat->masterNum; - pInfo->has_mnode = pDnode->mmgmt.deployed; -} - -static void dndSendMonitorReport(SDnode *pDnode) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); - - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - - SMonBasicInfo basicInfo = {0}; - dndGetMonitorBasicInfo(pDnode, &basicInfo); - monSetBasicInfo(pMonitor, &basicInfo); - - SMonClusterInfo clusterInfo = {0}; - SMonVgroupInfo vgroupInfo = {0}; - SMonGrantInfo grantInfo = {0}; - if (dndGetMnodeMonitorInfo(pDnode, &clusterInfo, &vgroupInfo, &grantInfo) == 0) { - monSetClusterInfo(pMonitor, &clusterInfo); - monSetVgroupInfo(pMonitor, &vgroupInfo); - monSetGrantInfo(pMonitor, &grantInfo); - } - - SMonDnodeInfo dnodeInfo = {0}; - dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); - monSetDnodeInfo(pMonitor, &dnodeInfo); - - SMonDiskInfo diskInfo = {0}; - if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { - monSetDiskInfo(pMonitor, &diskInfo); - } - - taosArrayDestroy(clusterInfo.dnodes); - taosArrayDestroy(clusterInfo.mnodes); - taosArrayDestroy(vgroupInfo.vgroups); - taosArrayDestroy(diskInfo.datadirs); - - monSendReport(pMonitor); - monCleanupMonitorInfo(pMonitor); -} - -static void *dnodeThreadRoutine(void *param) { - SDnode *pDnode = param; - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int64_t lastStatusTime = taosGetTimestampMs(); - int64_t lastMonitorTime = lastStatusTime; - - setThreadName("dnode-hb"); - - while (true) { - pthread_testcancel(); - taosMsleep(200); - if (dndGetStat(pDnode) != DND_STAT_RUNNING || pMgmt->dropped) { - continue; - } - - int64_t curTime = taosGetTimestampMs(); - - float statusInterval = (curTime - lastStatusTime) / 1000.0f; - if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { - dndSendStatusReq(pDnode); - lastStatusTime = curTime; - } - - float monitorInterval = (curTime - lastMonitorTime) / 1000.0f; - if (monitorInterval >= tsMonitorInterval) { - dndSendMonitorReport(pDnode); - lastMonitorTime = curTime; - } - } -} - -int32_t dndInitMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - pMgmt->dnodeId = 0; - pMgmt->rebootTime = taosGetTimestampMs(); - pMgmt->dropped = 0; - pMgmt->clusterId = 0; - taosInitRWLatch(&pMgmt->latch); - - char path[PATH_MAX]; - snprintf(path, PATH_MAX, "%s/dnode.json", pDnode->dir.dnode); - pMgmt->file = strdup(path); - if (pMgmt->file == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - if (pMgmt->dnodeHash == NULL) { - dError("failed to init dnode hash"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (dndReadDnodes(pDnode) != 0) { - dError("failed to read file:%s since %s", pMgmt->file, terrstr()); - return -1; - } - - if (pMgmt->dropped) { - dError("dnode not start since its already dropped"); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dndProcessMgmtQueue) != 0) { - dError("failed to start dnode mgmt worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dndProcessMgmtQueue) != 0) { - dError("failed to start dnode mgmt worker since %s", terrstr()); - return -1; - } - - pMgmt->threadId = taosCreateThread(dnodeThreadRoutine, pDnode); - if (pMgmt->threadId == NULL) { - dError("failed to init dnode thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dInfo("dnode-mgmt is initialized"); - return 0; -} - -void dndStopMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - dndCleanupWorker(&pMgmt->mgmtWorker); - dndCleanupWorker(&pMgmt->statusWorker); - - if (pMgmt->threadId != NULL) { - taosDestoryThread(pMgmt->threadId); - pMgmt->threadId = NULL; - } -} - -void dndCleanupMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - if (pMgmt->pDnodeEps != NULL) { - taosArrayDestroy(pMgmt->pDnodeEps); - pMgmt->pDnodeEps = NULL; - } - - if (pMgmt->dnodeHash != NULL) { - taosHashCleanup(pMgmt->dnodeHash); - pMgmt->dnodeHash = NULL; - } - - if (pMgmt->file != NULL) { - free(pMgmt->file); - pMgmt->file = NULL; - } - - taosWUnLockLatch(&pMgmt->latch); - dInfo("dnode-mgmt is cleaned up"); -} - -void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pEpSet && pEpSet->numOfEps > 0 && pMsg->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pDnode, pEpSet); - } - - SDnodeWorker *pWorker = &pMgmt->mgmtWorker; - if (pMsg->msgType == TDMT_MND_STATUS_RSP) { - pWorker = &pMgmt->statusWorker; - } - - if (dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)) != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } -} - -static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t code = 0; - - switch (pMsg->msgType) { - case TDMT_DND_CREATE_MNODE: - code = dndProcessCreateMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_ALTER_MNODE: - code = dndProcessAlterMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_MNODE: - code = dndProcessDropMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_QNODE: - code = dndProcessCreateQnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_QNODE: - code = dndProcessDropQnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_SNODE: - code = dndProcessCreateSnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_SNODE: - code = dndProcessDropSnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_BNODE: - code = dndProcessCreateBnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_BNODE: - code = dndProcessDropBnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CONFIG_DNODE: - code = dndProcessConfigDnodeReq(pDnode, pMsg); - break; - case TDMT_MND_STATUS_RSP: - dndProcessStatusRsp(pDnode, pMsg); - break; - case TDMT_MND_AUTH_RSP: - dndProcessAuthRsp(pDnode, pMsg); - break; - case TDMT_MND_GRANT_RSP: - dndProcessGrantRsp(pDnode, pMsg); - break; - case TDMT_DND_CREATE_VNODE: - code = dndProcessCreateVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_ALTER_VNODE: - code = dndProcessAlterVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_VNODE: - code = dndProcessDropVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_SYNC_VNODE: - code = dndProcessSyncVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_COMPACT_VNODE: - code = dndProcessCompactVnodeReq(pDnode, pMsg); - break; - default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - code = -1; - dError("RPC %p, dnode msg:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType)); - break; - } - - if (pMsg->msgType & 1u) { - if (code != 0) code = terrno; - SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle}; - rpcSendResponse(&rsp); - } - - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; - taosFreeQitem(pMsg); -} diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c deleted file mode 100644 index 0ea47c89d8cfa6af4307001a753fa0bfb00b09a2..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ /dev/null @@ -1,642 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndMnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg); -static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg); - -static SMnode *dndAcquireMnode(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SMnode *pMnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pMnode = pMgmt->pMnode; - } else { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pMnode != NULL) { - dTrace("acquire mnode, refCount:%d", refCount); - } - return pMnode; -} - -static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) { - if (pMnode == NULL) return; - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release mnode, refCount:%d", refCount); -} - -static int32_t dndReadMnodeFile(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - int32_t code = TSDB_CODE_DND_MNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_MNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_MNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_MNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_MNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_MNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); - if (!mnodes || mnodes->type != cJSON_Array) { - dError("failed to read %s since nodes not found", file); - goto PRASE_MNODE_OVER; - } - - pMgmt->replica = cJSON_GetArraySize(mnodes); - if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { - dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); - goto PRASE_MNODE_OVER; - } - - for (int32_t i = 0; i < pMgmt->replica; ++i) { - cJSON *node = cJSON_GetArrayItem(mnodes, i); - if (node == NULL) break; - - SReplica *pReplica = &pMgmt->replicas[i]; - - cJSON *id = cJSON_GetObjectItem(node, "id"); - if (!id || id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); - goto PRASE_MNODE_OVER; - } - pReplica->id = id->valueint; - - cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); - goto PRASE_MNODE_OVER; - } - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *port = cJSON_GetObjectItem(node, "port"); - if (!port || port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); - goto PRASE_MNODE_OVER; - } - pReplica->port = port->valueint; - } - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_MNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteMnodeFile(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); - for (int32_t i = 0; i < pMgmt->replica; ++i) { - SReplica *pReplica = &pMgmt->replicas[i]; - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); - if (i < pMgmt->replica - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartMnodeWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - if (dndInitWorker(pDnode, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode read worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode write worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode sync worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopMnodeWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 1) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->readWorker); - dndCleanupWorker(&pMgmt->writeWorker); - dndCleanupWorker(&pMgmt->syncWorker); -} - -static bool dndNeedDeployMnode(SDnode *pDnode) { - if (dndGetDnodeId(pDnode) > 0) { - return false; - } - - if (dndGetClusterId(pDnode) > 0) { - return false; - } - - if (strcmp(pDnode->cfg.localEp, pDnode->cfg.firstEp) != 0) { - return false; - } - - return true; -} - -static int32_t dndPutMsgToMWriteQ(SDnode *pDnode, SRpcMsg *pRpcMsg) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pRpcMsg); - return 0; -} - -static int32_t dndPutMsgToMReadQ(SDnode *pDnode, SRpcMsg* pRpcMsg) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pRpcMsg); - return 0; -} - -static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->putReqToMWriteQFp = dndPutMsgToMWriteQ; - pOption->putReqToMReadQFp = dndPutMsgToMReadQ; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); -} - -static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { - dndInitMnodeOption(pDnode, pOption); - pOption->replica = 1; - pOption->selfIndex = 0; - SReplica *pReplica = &pOption->replicas[0]; - pReplica->id = 1; - pReplica->port = pDnode->cfg.serverPort; - memcpy(pReplica->fqdn, pDnode->cfg.localFqdn, TSDB_FQDN_LEN); - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->selfIndex = pOption->selfIndex; - pMgmt->replica = pOption->replica; - memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); -} - -static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) { - dndInitMnodeOption(pDnode, pOption); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pOption->selfIndex = pMgmt->selfIndex; - pOption->replica = pMgmt->replica; - memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); -} - -static int32_t dndBuildMnodeOptionFromReq(SDnode *pDnode, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) { - dndInitMnodeOption(pDnode, pOption); - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - - pOption->replica = pCreate->replica; - pOption->selfIndex = -1; - for (int32_t i = 0; i < pCreate->replica; ++i) { - SReplica *pReplica = &pOption->replicas[i]; - pReplica->id = pCreate->replicas[i].id; - pReplica->port = pCreate->replicas[i].port; - memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); - if (pReplica->id == pOption->dnodeId) { - pOption->selfIndex = i; - } - } - - if (pOption->selfIndex == -1) { - dError("failed to build mnode options since %s", terrstr()); - return -1; - } - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->selfIndex = pOption->selfIndex; - pMgmt->replica = pOption->replica; - memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); - return 0; -} - -static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = mndOpen(pDnode->dir.mnode, pOption); - if (pMnode == NULL) { - dError("failed to open mnode since %s", terrstr()); - return -1; - } - - if (dndStartMnodeWorker(pDnode) != 0) { - dError("failed to start mnode worker since %s", terrstr()); - mndClose(pMnode); - mndDestroy(pDnode->dir.mnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteMnodeFile(pDnode) != 0) { - dError("failed to write mnode file since %s", terrstr()); - pMgmt->deployed = 0; - dndStopMnodeWorker(pDnode); - mndClose(pMnode); - mndDestroy(pDnode->dir.mnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pMnode = pMnode; - pMgmt->deployed = 1; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("mnode open successfully"); - return 0; -} - -static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOpt *pOption) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - if (mndAlter(pMnode, pOption) != 0) { - dError("failed to alter mnode since %s", terrstr()); - dndReleaseMnode(pDnode, pMnode); - return -1; - } - - dndReleaseMnode(pDnode, pMnode); - return 0; -} - -static int32_t dndDropMnode(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteMnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseMnode(pDnode, pMnode); - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - dndReleaseMnode(pDnode, pMnode); - dndStopMnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteMnodeFile(pDnode); - mndClose(pMnode); - pMgmt->pMnode = NULL; - mndDestroy(pDnode->dir.mnode); - - return 0; -} - - -int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateMnodeReq createReq = {0}; - if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.replica <= 1 || createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - SMnodeOpt option = {0}; - if (dndBuildMnodeOptionFromReq(pDnode, &option, &createReq) != 0) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - dndReleaseMnode(pDnode, pMnode); - terrno = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - dDebug("start to create mnode"); - return dndOpenMnode(pDnode, &option); -} - -int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDAlterMnodeReq alterReq = {0}; - if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (alterReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - SMnodeOpt option = {0}; - if (dndBuildMnodeOptionFromReq(pDnode, &option, &alterReq) != 0) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - dDebug("start to alter mnode"); - int32_t code = dndAlterMnode(pDnode, &option); - dndReleaseMnode(pDnode, pMnode); - - return code; -} - -int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropMnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - dDebug("start to drop mnode"); - int32_t code = dndDropMnode(pDnode); - dndReleaseMnode(pDnode, pMnode); - - return code; -} - -static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - mndProcessMsg(pMsg); - dndReleaseMnode(pDnode, pMnode); - } else { - mndSendRsp(pMsg, terrno); - } - - mndCleanupMsg(pMsg); -} - -static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg) { - int32_t code = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - code = dndWriteMsgToWorker(pWorker, pMsg, 0); - if (code != 0) code = terrno; - } - - if (code != 0) { - mndCleanupMsg(pMsg); - } - } - dndReleaseMnode(pDnode, pMnode); - - if (code != 0) { - if (pRpcMsg->msgType & 1u) { - if (code == TSDB_CODE_DND_MNODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) { - dndSendRedirectRsp(pDnode, pRpcMsg); - } else { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .ahandle = pRpcMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - } - rpcFreeCont(pRpcMsg->pCont); - } -} - -void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pMsg); -} - -void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.syncWorker, pMsg); -} - -void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pMsg); -} - -int32_t dndInitMnode(SDnode *pDnode) { - dInfo("dnode-mnode start to init"); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadMnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("mnode has been deployed and needs to be deleted"); - mndDestroy(pDnode->dir.mnode); - return 0; - } - - if (!pMgmt->deployed) { - bool needDeploy = dndNeedDeployMnode(pDnode); - if (!needDeploy) { - dDebug("mnode does not need to be deployed"); - return 0; - } - - dInfo("start to deploy mnode"); - SMnodeOpt option = {0}; - dndBuildMnodeDeployOption(pDnode, &option); - return dndOpenMnode(pDnode, &option); - } else { - dInfo("start to open mnode"); - SMnodeOpt option = {0}; - dndBuildMnodeOpenOption(pDnode, &option); - return dndOpenMnode(pDnode, &option); - } -} - -void dndCleanupMnode(SDnode *pDnode) { - dInfo("dnode-mnode start to clean up"); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - if (pMgmt->pMnode) { - dndStopMnodeWorker(pDnode); - mndClose(pMgmt->pMnode); - pMgmt->pMnode = NULL; - } - dInfo("dnode-mnode is cleaned up"); -} - -int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_APP_NOT_READY; - dTrace("failed to get user auth since %s", terrstr()); - return -1; - } - - int32_t code = mndRetriveAuth(pMnode, user, spi, encrypt, secret, ckey); - dndReleaseMnode(pDnode, pMnode); - - dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); - return code; -} - -int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) return -1; - - int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); - dndReleaseMnode(pDnode, pMnode); - return code; -} diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c deleted file mode 100644 index 93e2209610b98e957b956c5f9caa6babe8004e32..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndQnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static SQnode *dndAcquireQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SQnode *pQnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pQnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pQnode = pMgmt->pQnode; - } else { - terrno = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pQnode != NULL) { - dTrace("acquire qnode, refCount:%d", refCount); - } - return pQnode; -} - -static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode) { - if (pQnode == NULL) return; - - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release qnode, refCount:%d", refCount); -} - -static int32_t dndReadQnodeFile(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - int32_t code = TSDB_CODE_DND_QNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_QNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_QNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_QNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_QNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_QNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_QNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteQnodeFile(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartQnodeWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, dndProcessQnodeQueue) != 0) { - dError("failed to start qnode query worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, dndProcessQnodeQueue) != 0) { - dError("failed to start qnode fetch worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopQnodeWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->queryWorker); - dndCleanupWorker(&pMgmt->fetchWorker); -} - -static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - dndReleaseQnode(pDnode, pQnode); - terrno = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; - dError("failed to create qnode since %s", terrstr()); - return -1; - } - - SQnodeOpt option = {0}; - dndBuildQnodeOption(pDnode, &option); - - pQnode = qndOpen(&option); - if (pQnode == NULL) { - dError("failed to open qnode since %s", terrstr()); - return -1; - } - - if (dndStartQnodeWorker(pDnode) != 0) { - dError("failed to start qnode worker since %s", terrstr()); - qndClose(pQnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteQnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write qnode file since %s", terrstr()); - dndStopQnodeWorker(pDnode); - qndClose(pQnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pQnode = pQnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("qnode open successfully"); - return 0; -} - -static int32_t dndDropQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode == NULL) { - dError("failed to drop qnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteQnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseQnode(pDnode, pQnode); - dError("failed to drop qnode since %s", terrstr()); - return -1; - } - - dndReleaseQnode(pDnode, pQnode); - dndStopQnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteQnodeFile(pDnode); - qndClose(pQnode); - pMgmt->pQnode = NULL; - - return 0; -} - -int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateQnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION; - dError("failed to create qnode since %s", terrstr()); - return -1; - } else { - return dndOpenQnode(pDnode); - } -} - -int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropQnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION; - dError("failed to drop qnode since %s", terrstr()); - return -1; - } else { - return dndDropQnode(pDnode); - } -} - -static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SRpcMsg *pRsp = NULL; - int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - code = qndProcessMsg(pQnode, pMsg, &pRsp); - } - dndReleaseQnode(pDnode, pQnode); - - if (pMsg->msgType & 1u) { - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dndWriteQnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseQnode(pDnode, pQnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg); -} - -void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg); -} - -int32_t dndInitQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadQnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) return 0; - if (!pMgmt->deployed) return 0; - - return dndOpenQnode(pDnode); -} - -void dndCleanupQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - if (pMgmt->pQnode) { - dndStopQnodeWorker(pDnode); - qndClose(pMgmt->pQnode); - pMgmt->pQnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c deleted file mode 100644 index ea06c8c7519e48b881faa0f2e5aca8e64072dad3..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ /dev/null @@ -1,505 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndSnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -typedef struct { - int32_t vgId; - int32_t refCount; - int32_t snVersion; - int8_t dropped; - char *path; - SSnode *pImpl; - STaosQueue *pSharedQ; - STaosQueue *pUniqueQ; -} SSnodeObj; - -static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); - -static SSnode *dndAcquireSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - SSnode *pSnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pSnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pSnode = pMgmt->pSnode; - } else { - terrno = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pSnode != NULL) { - dTrace("acquire snode, refCount:%d", refCount); - } - return pSnode; -} - -static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) { - if (pSnode == NULL) return; - - SSnodeMgmt *pMgmt = &pDnode->smgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release snode, refCount:%d", refCount); -} - -static int32_t dndReadSnodeFile(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - int32_t code = TSDB_CODE_DND_SNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_SNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_SNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_SNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_SNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_SNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_SNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteSnodeFile(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartSnodeWorker(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); - for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { - SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker)); - if (pUniqueWorker == NULL) { - return -1; - } - if (dndInitWorker(pDnode, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) { - dError("failed to start snode unique worker since %s", terrstr()); - return -1; - } - taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker); - } - if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM, - SND_SHARED_THREAD_NUM, dndProcessSnodeSharedQueue)) { - dError("failed to start snode shared worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopSnodeWorker(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { - SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i); - dndCleanupWorker(worker); - } - taosArrayDestroy(pMgmt->uniqueWorkers); -} - -static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - dndReleaseSnode(pDnode, pSnode); - terrno = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; - dError("failed to create snode since %s", terrstr()); - return -1; - } - - SSnodeOpt option = {0}; - dndBuildSnodeOption(pDnode, &option); - - pSnode = sndOpen(pDnode->dir.snode, &option); - if (pSnode == NULL) { - dError("failed to open snode since %s", terrstr()); - return -1; - } - - if (dndStartSnodeWorker(pDnode) != 0) { - dError("failed to start snode worker since %s", terrstr()); - sndClose(pSnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteSnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write snode file since %s", terrstr()); - dndStopSnodeWorker(pDnode); - sndClose(pSnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pSnode = pSnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("snode open successfully"); - return 0; -} - -static int32_t dndDropSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode == NULL) { - dError("failed to drop snode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteSnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseSnode(pDnode, pSnode); - dError("failed to drop snode since %s", terrstr()); - return -1; - } - - dndReleaseSnode(pDnode, pSnode); - dndStopSnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteSnodeFile(pDnode); - sndClose(pSnode); - pMgmt->pSnode = NULL; - sndDestroy(pDnode->dir.snode); - - return 0; -} - -int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateSnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION; - dError("failed to create snode since %s", terrstr()); - return -1; - } else { - return dndOpenSnode(pDnode); - } -} - -int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropSnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION; - dError("failed to drop snode since %s", terrstr()); - return -1; - } else { - return dndDropSnode(pDnode); - } -} - -static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { - /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/ - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - for (int32_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - - sndProcessUMsg(pSnode, pMsg); - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - dndReleaseSnode(pDnode, pSnode); - } else { - for (int32_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - } -} - -static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) { - /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/ - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - sndProcessSMsg(pSnode, pMsg); - dndReleaseSnode(pDnode, pSnode); - } else { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - -#if 0 - if (pMsg->msgType & 1u) { - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } -#endif - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static FORCE_INLINE int32_t dndGetSWTypeFromMsg(SRpcMsg *pMsg) { - SStreamExecMsgHead *pHead = pMsg->pCont; - pHead->workerType = htonl(pHead->workerType); - return pHead->workerType; -} - -static FORCE_INLINE int32_t dndGetSWIdFromMsg(SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - pHead->streamTaskId = htonl(pHead->streamTaskId); - return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM; -} - -static void dndWriteSnodeMsgToWorkerByMsg(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - int32_t index = dndGetSWIdFromMsg(pMsg); - SDnodeWorker *pWorker = taosArrayGetP(pDnode->smgmt.uniqueWorkers, index); - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - - dndReleaseSnode(pDnode, pSnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -static void dndWriteSnodeMsgToMgmtWorker(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, 0); - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseSnode(pDnode, pSnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseSnode(pDnode, pSnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteSnodeMsgToMgmtWorker(pDnode, pMsg); -} - -void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - int32_t workerType = dndGetSWTypeFromMsg(pMsg); - if (workerType == SND_WORKER_TYPE__SHARED) { - dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg); - } else { - dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg); - } - } -} - -void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg); -} - -void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg); -} - -int32_t dndInitSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadSnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("snode has been deployed and needs to be deleted"); - sndDestroy(pDnode->dir.snode); - return 0; - } - - if (!pMgmt->deployed) return 0; - - return dndOpenSnode(pDnode); -} - -void dndCleanupSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - if (pMgmt->pSnode) { - dndStopSnodeWorker(pDnode); - sndClose(pMgmt->pSnode); - pMgmt->pSnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c deleted file mode 100644 index 617b6c0fc374678d29f16c5f74e1876c44aa4d01..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ /dev/null @@ -1,422 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/* this file is mainly responsible for the communication between DNODEs. Each - * dnode works as both server and client. Dnode may send status, grant, config - * messages to mnode, mnode may send create/alter/drop table/vnode messages - * to dnode. All theses messages are handled from here - */ - -#define _DEFAULT_SOURCE -#include "dndTransport.h" -#include "dndMgmt.h" -#include "dndMnode.h" -#include "dndSnode.h" -#include "dndVnodes.h" - -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" -#define INTERNAL_SECRET "_pwd" - -static void dndInitMsgFp(STransMgmt *pMgmt) { - // Requests handled by DNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_NETWORK_TEST)] = dndProcessMgmtMsg; - - // Requests handled by MNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONNECT)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONFIG_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_QNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_QNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_SNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_SNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_BNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_BNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TABLE_META)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_TRANS)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg; - - // Requests handled by VNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH_RSP)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONSUME)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_QUERY)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONNECT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_DISCONNECT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_RES_READY)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TASKS_STATUS)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CANCEL_TASK)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TASK)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_HEARTBEAT)] = dndProcessVnodeFetchMsg; - - // Requests handled by SNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_DEPLOY)] = dndProcessSnodeMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_EXEC)] = dndProcessSnodeExecMsg; -} - -static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode *pDnode = parent; - STransMgmt *pMgmt = &pDnode->tmgmt; - - tmsg_t msgType = pRsp->msgType; - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - if (pRsp == NULL || pRsp->pCont == NULL) return; - dTrace("RPC %p, rsp:%s ignored since dnode exiting, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - return; - } - - DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; - if (fp != NULL) { - dTrace("RPC %p, rsp:%s will be processed, code:0x%x app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF, - pRsp->ahandle); - (*fp)(pDnode, pRsp, pEpSet); - } else { - dError("RPC %p, rsp:%s not processed, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - } -} - -static int32_t dndInitClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "D-C"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dndProcessResponse; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = INTERNAL_USER; - rpcInit.ckey = INTERNAL_CKEY; - rpcInit.spi = 1; - rpcInit.parent = pDnode; - - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - rpcInit.secret = pass; - - pMgmt->clientRpc = rpcOpen(&rpcInit); - if (pMgmt->clientRpc == NULL) { - dError("failed to init dnode rpc client"); - return -1; - } - - dDebug("dnode rpc client is initialized"); - return 0; -} - -static void dndCleanupClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->clientRpc) { - rpcClose(pMgmt->clientRpc); - pMgmt->clientRpc = NULL; - dDebug("dnode rpc client is closed"); - } -} - -static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode *pDnode = param; - STransMgmt *pMgmt = &pDnode->tmgmt; - - tmsg_t msgType = pReq->msgType; - if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("RPC %p, network test req will be processed, app:%p", pReq->handle, pReq->ahandle); - dndProcessStartupReq(pDnode, pReq); - return; - } - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("RPC %p, req:%s ignored since dnode exiting, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) { - dError("RPC %p, req:%s ignored since dnode not running, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } - - if (pReq->pCont == NULL) { - dTrace("RPC %p, req:%s not processed since its empty, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - return; - } - - DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; - if (fp != NULL) { - dTrace("RPC %p, req:%s will be processed, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - (*fp)(pDnode, pReq, pEpSet); - } else { - dError("RPC %p, req:%s not processed since no handle, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - } -} - -static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { - STransMgmt *pMgmt = &pDnode->tmgmt; - - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); -} - -static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - if (strcmp(user, INTERNAL_USER) == 0) { - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - return 0; - } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - return 0; - } else { - return -1; - } -} - -static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SDnode *pDnode = parent; - - if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - if (terrno != TSDB_CODE_APP_NOT_READY) { - dTrace("failed to get user auth from mnode since %s", terrstr()); - return -1; - } - - SAuthReq authReq = {0}; - tstrncpy(authReq.user, user, TSDB_USER_LEN); - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); - tSerializeSAuthReq(pReq, contLen, &authReq); - - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; - SRpcMsg rpcRsp = {0}; - dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); - dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - terrno = rpcRsp.code; - dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); - } else { - SAuthRsp authRsp = {0}; - tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); - memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); - memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); - *spi = authRsp.spi; - *encrypt = authRsp.encrypt; - dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, - authRsp.encrypt); - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - -static int32_t dndInitServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - dndInitMsgFp(pMgmt); - - int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); - if (numOfThreads < 1) { - numOfThreads = 1; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = pDnode->cfg.serverPort; - rpcInit.label = "D-S"; - rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = tsMaxShellConns; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dndRetrieveUserAuthInfo; - rpcInit.parent = pDnode; - - pMgmt->serverRpc = rpcOpen(&rpcInit); - if (pMgmt->serverRpc == NULL) { - dError("failed to init dnode rpc server"); - return -1; - } - - dDebug("dnode rpc server is initialized"); - return 0; -} - -static void dndCleanupServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->serverRpc) { - rpcClose(pMgmt->serverRpc); - pMgmt->serverRpc = NULL; - dDebug("dnode rpc server is closed"); - } -} - -int32_t dndInitTrans(SDnode *pDnode) { - if (dndInitClient(pDnode) != 0) { - return -1; - } - - if (dndInitServer(pDnode) != 0) { - return -1; - } - - dInfo("dnode-transport is initialized"); - return 0; -} - -void dndCleanupTrans(SDnode *pDnode) { - dInfo("dnode-transport start to clean up"); - dndCleanupServer(pDnode); - dndCleanupClient(pDnode); - dInfo("dnode-transport is cleaned up"); -} - -int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->clientRpc == NULL) { - terrno = TSDB_CODE_DND_OFFLINE; - return -1; - } - - rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); - return 0; -} - -int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pReq) { - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - return dndSendReqToDnode(pDnode, &epSet, pReq); -} diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c deleted file mode 100644 index d311e1e41703e70ab0846d92f6946c81648b651c..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ /dev/null @@ -1,1024 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndVnodes.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "sync.h" - -typedef struct { - int32_t vgId; - int32_t vgVersion; - int8_t dropped; - uint64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; - char path[PATH_MAX + 20]; -} SWrapperCfg; - -typedef struct { - int32_t vgId; - int32_t refCount; - int32_t vgVersion; - int8_t dropped; - int8_t accessState; - uint64_t dbUid; - char *db; - char *path; - SVnode *pImpl; - STaosQueue *pWriteQ; - STaosQueue *pSyncQ; - STaosQueue *pApplyQ; - STaosQueue *pQueryQ; - STaosQueue *pFetchQ; -} SVnodeObj; - -typedef struct { - int32_t vnodeNum; - int32_t opened; - int32_t failed; - int32_t threadIndex; - pthread_t thread; - SDnode *pDnode; - SWrapperCfg *pCfgs; -} SVnodeThread; - -static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); - -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg); - -static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId); -static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl); -static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes); -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); -static int32_t dndWriteVnodesToFile(SDnode *pDnode); - -static int32_t dndOpenVnodes(SDnode *pDnode); -static void dndCloseVnodes(SDnode *pDnode); - -static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj *pVnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); - if (pVnode == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - } else { - refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - } - taosRUnLockLatch(&pMgmt->latch); - - if (pVnode != NULL) { - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - } - - return pVnode; -} - -static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { - if (pVnode == NULL) return; - - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); -} - -static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); - if (pVnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pVnode->vgId = pCfg->vgId; - pVnode->refCount = 0; - pVnode->dropped = 0; - pVnode->accessState = TSDB_VN_ALL_ACCCESS; - pVnode->pImpl = pImpl; - pVnode->vgVersion = pCfg->vgVersion; - pVnode->dbUid = pCfg->dbUid; - pVnode->db = tstrdup(pCfg->db); - pVnode->path = tstrdup(pCfg->path); - - if (pVnode->path == NULL || pVnode->db == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (dndAllocVnodeQueue(pDnode, pVnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - taosWLockLatch(&pMgmt->latch); - int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); - taosWUnLockLatch(&pMgmt->latch); - - if (code != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } - return code; -} - -static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosWLockLatch(&pMgmt->latch); - taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); - taosWUnLockLatch(&pMgmt->latch); - - dndReleaseVnode(pDnode, pVnode); - while (pVnode->refCount > 0) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); - - dndFreeVnodeQueue(pDnode, pVnode); - vnodeClose(pVnode->pImpl); - pVnode->pImpl = NULL; - - dDebug("vgId:%d, vnode is closed", pVnode->vgId); - - if (pVnode->dropped) { - dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped); - vnodeDestroy(pVnode->path); - } - - free(pVnode->path); - free(pVnode->db); - free(pVnode); -} - -static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosRLockLatch(&pMgmt->latch); - - int32_t num = 0; - int32_t size = taosHashGetSize(pMgmt->hash); - SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); - - void *pIter = taosHashIterate(pMgmt->hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - SVnodeObj *pVnode = *ppVnode; - if (pVnode && num < size) { - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - pVnodes[num] = (*ppVnode); - num++; - pIter = taosHashIterate(pMgmt->hash, pIter); - } else { - taosHashCancelIterate(pMgmt->hash, pIter); - } - } - - taosRUnLockLatch(&pMgmt->latch); - *numOfVnodes = num; - - return pVnodes; -} - -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 30000; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - FILE *fp = NULL; - char file[PATH_MAX + 20] = {0}; - SWrapperCfg *pCfgs = NULL; - - snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - - // fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_VNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_VNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_VNODE_OVER; - } - - cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); - if (!vnodes || vnodes->type != cJSON_Array) { - dError("failed to read %s since vnodes not found", file); - goto PRASE_VNODE_OVER; - } - - int32_t vnodesNum = cJSON_GetArraySize(vnodes); - if (vnodesNum > 0) { - pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); - if (pCfgs == NULL) { - dError("failed to read %s since out of memory", file); - goto PRASE_VNODE_OVER; - } - - for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); - SWrapperCfg *pCfg = &pCfgs[i]; - - cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_Number) { - dError("failed to read %s since vgId not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->vgId = vgId->valueint; - snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId); - - cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->dropped = dropped->valueint; - - cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); - if (!vgVersion || vgVersion->type != cJSON_Number) { - dError("failed to read %s since vgVersion not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->vgVersion = vgVersion->valueint; - - cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); - if (!dbUid || dbUid->type != cJSON_String) { - dError("failed to read %s since dbUid not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->dbUid = atoll(dbUid->valuestring); - - cJSON *db = cJSON_GetObjectItem(vnode, "db"); - if (!db || db->type != cJSON_String) { - dError("failed to read %s since db not found", file); - goto PRASE_VNODE_OVER; - } - tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN); - } - - *ppCfgs = pCfgs; - } - - *numOfVnodes = vnodesNum; - code = 0; - dInfo("succcessed to read file %s", file); - -PRASE_VNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - return code; -} - -static int32_t dndWriteVnodesToFile(SDnode *pDnode) { - char file[PATH_MAX + 20] = {0}; - char realfile[PATH_MAX + 20] = {0}; - snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); - snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); - - int32_t len = 0; - int32_t maxLen = 65536; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - len += snprintf(content + len, maxLen - len, " {\n"); - len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); - len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); - len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); - len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); - if (i < numOfVnodes - 1) { - len += snprintf(content + len, maxLen - len, " },\n"); - } else { - len += snprintf(content + len, maxLen - len, " }\n"); - } - } - len += snprintf(content + len, maxLen - len, " ]\n"); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - terrno = 0; - - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - dndReleaseVnode(pDnode, pVnode); - } - - if (pVnodes != NULL) { - free(pVnodes); - } - - dDebug("successed to write %s", realfile); - return taosRenameFile(file, realfile); -} - -static void *dnodeOpenVnodeFunc(void *param) { - SVnodeThread *pThread = param; - SDnode *pDnode = pThread->pDnode; - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); - setThreadName("open-vnodes"); - - for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - SWrapperCfg *pCfg = &pThread->pCfgs[v]; - - char stepDesc[TSDB_STEP_DESC_LEN] = {0}; - snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, - pMgmt->stat.openVnodes, pMgmt->stat.totalVnodes); - dndReportStartup(pDnode, "open-vnodes", stepDesc); - - SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; - SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); - if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); - pThread->failed++; - } else { - dndOpenVnode(pDnode, pCfg, pImpl); - dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); - pThread->opened++; - } - - atomic_add_fetch_32(&pMgmt->stat.openVnodes, 1); - } - - dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); - return NULL; -} - -static int32_t dndOpenVnodes(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosInitRWLatch(&pMgmt->latch); - - pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - if (pMgmt->hash == NULL) { - dError("failed to init vnode hash"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - SWrapperCfg *pCfgs = NULL; - int32_t numOfVnodes = 0; - if (dndGetVnodesFromFile(pDnode, &pCfgs, &numOfVnodes) != 0) { - dInfo("failed to get vnode list from disk since %s", terrstr()); - return -1; - } - - pMgmt->stat.totalVnodes = numOfVnodes; - - int32_t threadNum = tsNumOfCores; -#if 1 - threadNum = 1; -#endif - - int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - - SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); - for (int32_t t = 0; t < threadNum; ++t) { - threads[t].threadIndex = t; - threads[t].pDnode = pDnode; - threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); - } - - for (int32_t v = 0; v < numOfVnodes; ++v) { - int32_t t = v % threadNum; - SVnodeThread *pThread = &threads[t]; - pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; - } - - dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); - - for (int32_t t = 0; t < threadNum; ++t) { - SVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum == 0) continue; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnodeFunc, pThread) != 0) { - dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - } - - for (int32_t t = 0; t < threadNum; ++t) { - SVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { - pthread_join(pThread->thread, NULL); - } - free(pThread->pCfgs); - } - free(threads); - free(pCfgs); - - if (pMgmt->stat.openVnodes != pMgmt->stat.totalVnodes) { - dError("there are total vnodes:%d, opened:%d", pMgmt->stat.totalVnodes, pMgmt->stat.openVnodes); - return -1; - } else { - dInfo("total vnodes:%d open successfully", pMgmt->stat.totalVnodes); - return 0; - } -} - -static void dndCloseVnodes(SDnode *pDnode) { - dInfo("start to close all vnodes"); - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); - - for (int32_t i = 0; i < numOfVnodes; ++i) { - dndCloseVnode(pDnode, pVnodes[i]); - } - - if (pVnodes != NULL) { - free(pVnodes); - } - - if (pMgmt->hash != NULL) { - taosHashCleanup(pMgmt->hash); - pMgmt->hash = NULL; - } - - dInfo("total vnodes:%d are all closed", numOfVnodes); -} - -static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { - pCfg->vgId = pCreate->vgId; - pCfg->wsize = pCreate->cacheBlockSize; - pCfg->ssize = pCreate->cacheBlockSize; - pCfg->lsize = pCreate->cacheBlockSize; - pCfg->isHeapAllocator = true; - pCfg->ttl = 4; - pCfg->keep = pCreate->daysToKeep0; - pCfg->streamMode = pCreate->streamMode; - pCfg->isWeak = true; - pCfg->tsdbCfg.keep = pCreate->daysToKeep0; - pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; - pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; - pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; - pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; - pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; - pCfg->walCfg.level = pCreate->walLevel; - pCfg->walCfg.retentionPeriod = 10; - pCfg->walCfg.retentionSize = 128; - pCfg->walCfg.rollPeriod = 128; - pCfg->walCfg.segSize = 128; - pCfg->walCfg.vgId = pCreate->vgId; - pCfg->hashBegin = pCreate->hashBegin; - pCfg->hashEnd = pCreate->hashEnd; - pCfg->hashMethod = pCreate->hashMethod; -} - -static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { - memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN); - pCfg->dbUid = pCreate->dbUid; - pCfg->dropped = 0; - snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId); - pCfg->vgId = pCreate->vgId; - pCfg->vgVersion = pCreate->vgVersion; -} - -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SCreateVnodeReq createReq = {0}; - if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - dDebug("vgId:%d, create vnode req is received", createReq.vgId); - - SVnodeCfg vnodeCfg = {0}; - dndGenerateVnodeCfg(&createReq, &vnodeCfg); - - SWrapperCfg wrapperCfg = {0}; - dndGenerateWrapperCfg(pDnode, &createReq, &wrapperCfg); - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; - dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); - return -1; - } - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, createReq.vgId); - if (pVnode != NULL) { - dDebug("vgId:%d, already exist", createReq.vgId); - dndReleaseVnode(pDnode, pVnode); - terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; - return -1; - } - - vnodeCfg.pDnode = pDnode; - vnodeCfg.pTfs = pDnode->pTfs; - vnodeCfg.dbId = wrapperCfg.dbUid; - SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); - if (pImpl == NULL) { - dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); - return -1; - } - - int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); - if (code != 0) { - dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); - vnodeClose(pImpl); - vnodeDestroy(wrapperCfg.path); - terrno = code; - return code; - } - - code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(wrapperCfg.path); - terrno = code; - return code; - } - - return 0; -} - -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SAlterVnodeReq alterReq = {0}; - if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - dDebug("vgId:%d, alter vnode req is received", alterReq.vgId); - - SVnodeCfg vnodeCfg = {0}; - dndGenerateVnodeCfg(&alterReq, &vnodeCfg); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, alterReq.vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); - return -1; - } - - if (alterReq.vgVersion == pVnode->vgVersion) { - dndReleaseVnode(pDnode, pVnode); - dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId); - return 0; - } - - if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { - dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - int32_t oldVersion = pVnode->vgVersion; - pVnode->vgVersion = alterReq.vgVersion; - int32_t code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - pVnode->vgVersion = oldVersion; - } - - dndReleaseVnode(pDnode, pVnode); - return code; -} - -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDropVnodeReq dropReq = {0}; - if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - int32_t vgId = dropReq.vgId; - dDebug("vgId:%d, drop vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; - return -1; - } - - pVnode->dropped = 1; - if (dndWriteVnodesToFile(pDnode) != 0) { - pVnode->dropped = 0; - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndCloseVnode(pDnode, pVnode); - dndWriteVnodesToFile(pDnode); - - return 0; -} - -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SSyncVnodeReq syncReq = {0}; - tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq); - - int32_t vgId = syncReq.vgId; - dDebug("vgId:%d, sync vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); - return -1; - } - - if (vnodeSync(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndReleaseVnode(pDnode, pVnode); - return 0; -} - -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SCompactVnodeReq compatcReq = {0}; - tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq); - - int32_t vgId = compatcReq.vgId; - dDebug("vgId:%d, compact vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); - return -1; - } - - if (vnodeCompact(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndReleaseVnode(pDnode, pVnode); - return 0; -} - -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessQueryMsg(pVnode->pImpl, pMsg); } - -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessFetchMsg(pVnode->pImpl, pMsg); } - -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - void *ptr = taosArrayPush(pArray, &pMsg); - assert(ptr != NULL); - } - - vnodeProcessWMsgs(pVnode->pImpl, pArray); - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pRsp = NULL; - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - int32_t code = vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - - taosArrayDestroy(pArray); -} - -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SRpcMsg *pMsg = NULL; - - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pMsg); - - // todo - SRpcMsg *pRsp = NULL; - (void)vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); - } -} - -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SRpcMsg *pMsg = NULL; - - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pMsg); - - // todo - SRpcMsg *pRsp = NULL; - (void)vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp); - } -} - -static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg, bool sendRsp) { - int32_t code = 0; - - if (pQueue == NULL) { - code = TSDB_CODE_MSG_NOT_PROCESSED; - } else { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - *pMsg = *pRpcMsg; - if (taosWriteQitem(pQueue, pMsg) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } - } - - if (code != TSDB_CODE_SUCCESS && sendRsp) { - if (pRpcMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pRpcMsg->pCont); - } - - return code; -} - -static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - pHead->contLen = htonl(pHead->contLen); - pHead->vgId = htonl(pHead->vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); - if (pVnode == NULL) { - dError("vgId:%d, failed to acquire vnode while process req", pHead->vgId); - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } - - return pVnode; -} - -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - // pHead->vgId = htonl(pHead->vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); - if (pVnode == NULL) return -1; - - int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, false); - dndReleaseVnode(pDnode, pVnode); - return code; -} - -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) return -1; - - int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg); - dndReleaseVnode(pDnode, pVnode); - return code; -} - -static int32_t dndInitVnodeWorkers(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - int32_t maxFetchThreads = 4; - int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); - int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); - int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); - int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); - - SQWorkerPool *pQPool = &pMgmt->queryPool; - pQPool->name = "vnode-query"; - pQPool->min = minQueryThreads; - pQPool->max = maxQueryThreads; - if (tQWorkerInit(pQPool) != 0) return -1; - - SFWorkerPool *pFPool = &pMgmt->fetchPool; - pFPool->name = "vnode-fetch"; - pFPool->min = minFetchThreads; - pFPool->max = maxFetchThreads; - if (tFWorkerInit(pFPool) != 0) return -1; - - SWWorkerPool *pWPool = &pMgmt->writePool; - pWPool->name = "vnode-write"; - pWPool->max = maxWriteThreads; - if (tWWorkerInit(pWPool) != 0) return -1; - - pWPool = &pMgmt->syncPool; - pWPool->name = "vnode-sync"; - pWPool->max = maxSyncThreads; - if (tWWorkerInit(pWPool) != 0) return -1; - - dDebug("vnode workers is initialized"); - return 0; -} - -static void dndCleanupVnodeWorkers(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tFWorkerCleanup(&pMgmt->fetchPool); - tQWorkerCleanup(&pMgmt->queryPool); - tWWorkerCleanup(&pMgmt->writePool); - tWWorkerCleanup(&pMgmt->syncPool); - dDebug("vnode workers is closed"); -} - -static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeWriteQueue); - pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeApplyQueue); - pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)dndProcessVnodeSyncQueue); - pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)dndProcessVnodeFetchQueue); - pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)dndProcessVnodeQueryQueue); - - if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || - pVnode->pQueryQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); - tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); - tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); - tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); - pVnode->pWriteQ = NULL; - pVnode->pApplyQ = NULL; - pVnode->pSyncQ = NULL; - pVnode->pFetchQ = NULL; - pVnode->pQueryQ = NULL; -} - -int32_t dndInitVnodes(SDnode *pDnode) { - dInfo("dnode-vnodes start to init"); - - if (dndInitVnodeWorkers(pDnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - dError("failed to init vnode workers since %s", terrstr()); - return -1; - } - - if (dndOpenVnodes(pDnode) != 0) { - dError("failed to open vnodes since %s", terrstr()); - return -1; - } - - dInfo("dnode-vnodes is initialized"); - return 0; -} - -void dndCleanupVnodes(SDnode *pDnode) { - dInfo("dnode-vnodes start to clean up"); - dndCloseVnodes(pDnode); - dndCleanupVnodeWorkers(pDnode); - dInfo("dnode-vnodes is cleaned up"); -} - -void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodesStat *pStat = &pMgmt->stat; - int32_t totalVnodes = 0; - int32_t masterNum = 0; - int64_t numOfSelectReqs = 0; - int64_t numOfInsertReqs = 0; - int64_t numOfInsertSuccessReqs = 0; - int64_t numOfBatchInsertReqs = 0; - int64_t numOfBatchInsertSuccessReqs = 0; - - taosRLockLatch(&pMgmt->latch); - - void *pIter = taosHashIterate(pMgmt->hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - if (ppVnode == NULL || *ppVnode == NULL) continue; - - SVnodeObj *pVnode = *ppVnode; - SVnodeLoad vload = {0}; - vnodeGetLoad(pVnode->pImpl, &vload); - taosArrayPush(pLoads, &vload); - - numOfSelectReqs += vload.numOfSelectReqs; - numOfInsertReqs += vload.numOfInsertReqs; - numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs; - numOfBatchInsertReqs += vload.numOfBatchInsertReqs; - numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs; - totalVnodes++; - if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++; - - pIter = taosHashIterate(pMgmt->hash, pIter); - } - - taosRUnLockLatch(&pMgmt->latch); - - pStat->totalVnodes = totalVnodes; - pStat->masterNum = masterNum; - pStat->numOfSelectReqs = numOfSelectReqs; - pStat->numOfInsertReqs = numOfInsertReqs; - pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs; - pStat->numOfBatchInsertReqs = numOfBatchInsertReqs; - pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; -} diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/impl/src/dndWorker.c deleted file mode 100644 index 38f8737b2bbbc0796a98688498eeab7d44cdef76..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndWorker.h" - -int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, - int32_t maxNum, void *queueFp) { - if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - return -1; - } - - pWorker->type = type; - pWorker->name = name; - pWorker->minNum = minNum; - pWorker->maxNum = maxNum; - pWorker->queueFp = queueFp; - pWorker->pDnode = pDnode; - - if (pWorker->type == DND_WORKER_SINGLE) { - SQWorkerPool *pPool = &pWorker->pool; - pPool->name = name; - pPool->min = minNum; - pPool->max = maxNum; - if (tQWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pWorker->queue = tQWorkerAllocQueue(pPool, pDnode, (FItem)queueFp); - if (pWorker->queue == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - } else if (pWorker->type == DND_WORKER_MULTI) { - SWWorkerPool *pPool = &pWorker->mpool; - pPool->name = name; - pPool->max = maxNum; - if (tWWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pWorker->queue = tWWorkerAllocQueue(pPool, pDnode, (FItems)queueFp); - if (pWorker->queue == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - } else { - terrno = TSDB_CODE_INVALID_PARA; - } - - return 0; -} - -void dndCleanupWorker(SDnodeWorker *pWorker) { - while (!taosQueueEmpty(pWorker->queue)) { - taosMsleep(10); - } - - if (pWorker->type == DND_WORKER_SINGLE) { - tQWorkerCleanup(&pWorker->pool); - tQWorkerFreeQueue(&pWorker->pool, pWorker->queue); - } else if (pWorker->type == DND_WORKER_MULTI) { - tWWorkerCleanup(&pWorker->mpool); - tWWorkerFreeQueue(&pWorker->mpool, pWorker->queue); - } else { - } -} - -int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) { - if (pWorker == NULL || pWorker->queue == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - return -1; - } - - void *pMsg = NULL; - if (contLen != 0) { - pMsg = taosAllocateQitem(contLen); - if (pMsg != NULL) { - memcpy(pMsg, pCont, contLen); - } - } else { - pMsg = pCont; - } - - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (taosWriteQitem(pWorker->queue, pMsg) != 0) { - if (contLen != 0) { - taosFreeQitem(pMsg); - } - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/main/CMakeLists.txt similarity index 62% rename from source/dnode/mgmt/daemon/CMakeLists.txt rename to source/dnode/mgmt/main/CMakeLists.txt index 3238bbf3f07279e590350d060bb1a8d4ad47c7ef..baa486b91fe3d039a127b47c76576ded80d3c1b9 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/main/CMakeLists.txt @@ -1,5 +1,5 @@ -aux_source_directory(src DAEMON_SRC) -add_executable(taosd ${DAEMON_SRC}) +aux_source_directory(src EXEC_SRC) +add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndMain.h similarity index 58% rename from source/dnode/mgmt/impl/inc/dndInt.h rename to source/dnode/mgmt/main/inc/dndMain.h index a8530037daabecd6c2fad43f8ad7f680315bd6e4..1958d628a0ad2ba2a2c28c4a835ee6a5ff13f37e 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndMain.h @@ -1,3 +1,4 @@ + /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -13,39 +14,20 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_INT_H_ -#define _TD_DND_INT_H_ +#ifndef _TD_DND_MAIN_H_ +#define _TD_DND_MAIN_H_ -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" +#include "dnode.h" -#include "cJSON.h" -#include "monitor.h" -#include "tcache.h" -#include "tcrc32c.h" -#include "tdatablock.h" +#include "taoserror.h" +#include "tconfig.h" #include "tglobal.h" -#include "thash.h" -#include "tlockfree.h" #include "tlog.h" -#include "tmsg.h" -#include "tqueue.h" -#include "trpc.h" -#include "tthread.h" -#include "ttime.h" -#include "tworker.h" +#include "version.h" -#include "dnode.h" - -#include "bnode.h" -#include "mnode.h" -#include "qnode.h" -#include "snode.h" -#include "vnode.h" -#include "tfs.h" +#ifdef __cplusplus +extern "C" { +#endif #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} @@ -54,20 +36,13 @@ extern "C" { #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} -typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; -typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; -typedef enum { DND_ENV_INIT = 0, DND_ENV_READY = 1, DND_ENV_CLEANUP = 2 } EEnvStat; -typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); - -EStat dndGetStat(SDnode *pDnode); -void dndSetStat(SDnode *pDnode, EStat stat); -const char *dndStatStr(EStat stat); - -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); +void dndDumpCfg(); +void dndPrintVersion(); +void dndGenerateGrant(); +SDnodeOpt dndGetOpt(); #ifdef __cplusplus } #endif -#endif /*_TD_DND_INT_H_*/ +#endif /*_TD_DND_MAIN_H_*/ diff --git a/source/dnode/mgmt/main/src/dndMain.c b/source/dnode/mgmt/main/src/dndMain.c new file mode 100644 index 0000000000000000000000000000000000000000..3aff3446dac278ff38636c6dfca633b25c0a2afd --- /dev/null +++ b/source/dnode/mgmt/main/src/dndMain.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndMain.h" + +static struct { + bool dumpConfig; + bool generateGrant; + bool printAuth; + bool printVersion; + char envFile[PATH_MAX]; + char apolloUrl[PATH_MAX]; + SDnode *pDnode; +} global = {0}; + +static void dndSigintHandle(int signum, void *info, void *ctx) { + dInfo("signal:%d is received", signum); + SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode); + if (pDnode != NULL) { + dndHandleEvent(pDnode, DND_EVENT_STOP); + } +} + +static void dndSetSignalHandle() { + taosSetSignal(SIGTERM, dndSigintHandle); + taosSetSignal(SIGHUP, dndSigintHandle); + taosSetSignal(SIGINT, dndSigintHandle); + taosSetSignal(SIGABRT, dndSigintHandle); + taosSetSignal(SIGBREAK, dndSigintHandle); +} + +static int32_t dndParseOption(int32_t argc, char const *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-C") == 0) { + global.dumpConfig = true; + } else if (strcmp(argv[i], "-k") == 0) { + global.generateGrant = true; + } else if (strcmp(argv[i], "-V") == 0) { + global.printVersion = true; + } else { + } + } + + return 0; +} + +static int32_t dndRunDnode() { + if (dndInit() != 0) { + dInfo("failed to initialize dnode environment since %s", terrstr()); + return -1; + } + + SDnodeOpt option = dndGetOpt(); + + SDnode *pDnode = dndCreate(&option); + if (pDnode == NULL) { + dError("failed to to create dnode object since %s", terrstr()); + return -1; + } else { + global.pDnode = pDnode; + dndSetSignalHandle(); + } + + dInfo("start the TDengine service"); + int32_t code = dndRun(pDnode); + dInfo("start shutting down the TDengine service"); + + global.pDnode = NULL; + dndClose(pDnode); + dndCleanup(); + taosCloseLog(); + taosCleanupCfg(); + return code; +} + +int main(int argc, char const *argv[]) { + if (!taosCheckSystemIsSmallEnd()) { + dError("failed to start TDengine since on non-small-end machines"); + return -1; + } + + if (dndParseOption(argc, argv) != 0) { + return -1; + } + + if (global.generateGrant) { + dndGenerateGrant(); + return 0; + } + + if (global.printVersion) { + dndPrintVersion(); + return 0; + } + + if (taosCreateLog("taosdlog", 1, configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) { + dError("failed to start TDengine since read log config error"); + return -1; + } + + if (taosInitCfg(configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) { + dError("failed to start TDengine since read config error"); + return -1; + } + + if (global.dumpConfig) { + dndDumpCfg(); + taosCleanupCfg(); + taosCloseLog(); + return 0; + } + + return dndRunDnode(); +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/main/src/dndUtil.c similarity index 55% rename from source/dnode/mgmt/daemon/src/dmnUtil.c rename to source/dnode/mgmt/main/src/dndUtil.c index d58a5968c0726ae99fd23bb5116bfa3269da1dd7..e07ef68c774fb70f789cd00d43d37a0a21b504f9 100644 --- a/source/dnode/mgmt/daemon/src/dmnUtil.c +++ b/source/dnode/mgmt/main/src/dndUtil.c @@ -14,15 +14,15 @@ */ #define _DEFAULT_SOURCE -#include "dmnInt.h" +#include "dndMain.h" -void dmnGenerateGrant() { +void dndGenerateGrant() { #if 0 grantParseParameter(); #endif } -void dmnPrintVersion() { +void dndPrintVersion() { #ifdef TD_ENTERPRISE char *releaseName = "enterprise"; #else @@ -32,3 +32,24 @@ void dmnPrintVersion() { printf("gitinfo: %s\n", gitinfo); printf("builuInfo: %s\n", buildinfo); } + +void dndDumpCfg() { + SConfig *pCfg = taosGetCfg(); + cfgDumpCfg(pCfg, 0, 1); +} + +SDnodeOpt dndGetOpt() { + SConfig *pCfg = taosGetCfg(); + SDnodeOpt option = {0}; + + option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir)); + tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp)); + tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp)); + option.serverPort = tsServerPort; + tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn)); + snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort); + option.pDisks = tsDiskCfg; + option.numOfDisks = tsDiskCfgNum; + return option; +} diff --git a/source/dnode/mgmt/mnode/inc/mm.h b/source/dnode/mgmt/mnode/inc/mm.h new file mode 100644 index 0000000000000000000000000000000000000000..6ed6c42d93c7bceacba6c386158abaef38b74baa --- /dev/null +++ b/source/dnode/mgmt/mnode/inc/mm.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_MNODE_H_ +#define _TD_DND_MNODE_H_ + +#include "dnd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void mmGetMgmtFp(SMgmtWrapper *pMgmt); + +int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mnode/inc/mmInt.h b/source/dnode/mgmt/mnode/inc/mmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..cd4585048b90cc3b4d11fe00b9abbd220d9632dd --- /dev/null +++ b/source/dnode/mgmt/mnode/inc/mmInt.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_MNODE_INT_H_ +#define _TD_DND_MNODE_INT_H_ + +#include "mm.h" +#include "mnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SMnodeMgmt { + SMnode *pMnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SSingleWorker queryWorker; + SSingleWorker readWorker; + SSingleWorker writeWorker; + SSingleWorker syncWorker; + SReplica replicas[TSDB_MAX_REPLICA]; + int8_t replica; + int8_t selfIndex; +} SMnodeMgmt; + +// mmFile.c +int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed); +int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed); + +// mmInt.c +int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq); +int32_t mmDrop(SMgmtWrapper *pWrapper); +int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq); + +// mmMsg.c +void mmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); + +// mmWorker.c +int32_t mmStartWorker(SMnodeMgmt *pMgmt); +void mmStopWorker(SMnodeMgmt *pMgmt); +int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessQueryMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg); +int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg); +int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_MNODE_INT_H_*/ diff --git a/source/dnode/mgmt/mnode/src/mmFile.c b/source/dnode/mgmt/mnode/src/mmFile.c new file mode 100644 index 0000000000000000000000000000000000000000..e5cc0ce0871bfa398bd7c46f4c9260bb16e8f0de --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmFile.c @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 4096; + char *content = taosMemoryCalloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_MNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_MNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_MNODE_OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", file); + goto PRASE_MNODE_OVER; + } + *pDeployed = deployed->valueint; + + cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); + if (!mnodes || mnodes->type != cJSON_Array) { + dError("failed to read %s since nodes not found", file); + goto PRASE_MNODE_OVER; + } + + pMgmt->replica = cJSON_GetArraySize(mnodes); + if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { + dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); + goto PRASE_MNODE_OVER; + } + + for (int32_t i = 0; i < pMgmt->replica; ++i) { + cJSON *node = cJSON_GetArrayItem(mnodes, i); + if (node == NULL) break; + + SReplica *pReplica = &pMgmt->replicas[i]; + + cJSON *id = cJSON_GetObjectItem(node, "id"); + if (!id || id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); + goto PRASE_MNODE_OVER; + } + pReplica->id = id->valueint; + + cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); + if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto PRASE_MNODE_OVER; + } + tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *port = cJSON_GetObjectItem(node, "port"); + if (!port || port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto PRASE_MNODE_OVER; + } + pReplica->port = port->valueint; + } + + code = 0; + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + +PRASE_MNODE_OVER: + if (content != NULL) taosMemoryFree(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno);; + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 4096; + char *content = taosMemoryCalloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", deployed); + len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); + for (int32_t i = 0; i < pMgmt->replica; ++i) { + SReplica *pReplica = &pMgmt->replicas[i]; + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); + if (i < pMgmt->replica - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + taosMemoryFree(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno);; + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d", realfile, deployed); + return 0; +} diff --git a/source/dnode/mgmt/mnode/src/mmInt.c b/source/dnode/mgmt/mnode/src/mmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..61afcb11d15b77b6ac3c87bbb02ab9f55a23b097 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmInt.c @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" +#include "wal.h" + +static bool mmDeployRequired(SDnode *pDnode) { + if (pDnode->dnodeId > 0) return false; + if (pDnode->clusterId > 0) return false; + if (strcmp(pDnode->localEp, pDnode->firstEp) != 0) return false; + return true; +} + +static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) { + SMnodeMgmt mgmt = {0}; + mgmt.path = pWrapper->path; + if (mmReadFile(&mgmt, required) != 0) { + return -1; + } + + if (!(*required)) { + *required = mmDeployRequired(pWrapper->pDnode); + } + + return 0; +} + +static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; + + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToQueryQueue; + msgCb.queueFps[READ_QUEUE] = mmPutMsgToReadQueue; + msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + pOption->msgCb = msgCb; +} + +static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + + mmInitOption(pMgmt, pOption); + pOption->replica = 1; + pOption->selfIndex = 0; + SReplica *pReplica = &pOption->replicas[0]; + pReplica->id = 1; + pReplica->port = pDnode->serverPort; + tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN); + + pMgmt->selfIndex = pOption->selfIndex; + pMgmt->replica = pOption->replica; + memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); +} + +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + mmInitOption(pMgmt, pOption); + pOption->selfIndex = pMgmt->selfIndex; + pOption->replica = pMgmt->replica; + memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); +} + +static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) { + mmInitOption(pMgmt, pOption); + + pOption->replica = pCreate->replica; + pOption->selfIndex = -1; + for (int32_t i = 0; i < pCreate->replica; ++i) { + SReplica *pReplica = &pOption->replicas[i]; + pReplica->id = pCreate->replicas[i].id; + pReplica->port = pCreate->replicas[i].port; + memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); + if (pReplica->id == pOption->dnodeId) { + pOption->selfIndex = i; + } + } + + if (pOption->selfIndex == -1) { + dError("failed to build mnode options since %s", terrstr()); + return -1; + } + + pMgmt->selfIndex = pOption->selfIndex; + pMgmt->replica = pOption->replica; + memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); + return 0; +} + +static int32_t mmOpenImp(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq) { + SMnodeOpt option = {0}; + if (pReq != NULL) { + if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) { + return -1; + } + } else { + bool deployed = false; + if (mmReadFile(pMgmt, &deployed) != 0) { + dError("failed to read file since %s", terrstr()); + return -1; + } + + if (!deployed) { + dInfo("mnode start to deploy"); + mmBuildOptionForDeploy(pMgmt, &option); + } else { + dInfo("mnode start to open"); + mmBuildOptionForOpen(pMgmt, &option); + } + } + + pMgmt->pMnode = mndOpen(pMgmt->path, &option); + if (pMgmt->pMnode == NULL) { + dError("failed to open mnode since %s", terrstr()); + return -1; + } + + if (mmStartWorker(pMgmt) != 0) { + dError("failed to start mnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (mmWriteFile(pMgmt, deployed) != 0) { + dError("failed to write mnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void mmCloseImp(SMnodeMgmt *pMgmt) { + if (pMgmt->pMnode != NULL) { + mmStopWorker(pMgmt); + mndClose(pMgmt->pMnode); + pMgmt->pMnode = NULL; + } +} + +int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq) { + SMnodeOpt option = {0}; + if (pReq != NULL) { + if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) { + return -1; + } + } + + return mndAlter(pMgmt->pMnode, &option); +} + +int32_t mmDrop(SMgmtWrapper *pWrapper) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("mnode-mgmt start to drop"); + bool deployed = false; + if (mmWriteFile(pMgmt, deployed) != 0) { + dError("failed to drop mnode since %s", terrstr()); + return -1; + } + + mmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + taosMemoryFree(pMgmt); + dInfo("mnode-mgmt is dropped"); + return 0; +} + +static void mmClose(SMgmtWrapper *pWrapper) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("mnode-mgmt start to cleanup"); + mmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + taosMemoryFree(pMgmt); + dInfo("mnode-mgmt is cleaned up"); +} + +int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) { + dInfo("mnode-mgmt start to init"); + if (walInit() != 0) { + dError("failed to init wal since %s", terrstr()); + return -1; + } + + SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = mmOpenImp(pMgmt, pReq); + if (code != 0) { + dError("failed to init mnode-mgmt since %s", terrstr()); + mmClose(pWrapper); + } else { + dInfo("mnode-mgmt is initialized"); + } + + return code; +} + +static int32_t mmOpen(SMgmtWrapper *pWrapper) { + return mmOpenFromMsg(pWrapper, NULL); +} + +static int32_t mmStart(SMgmtWrapper *pWrapper) { + dDebug("mnode-mgmt start to run"); + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mndStart(pMgmt->pMnode); +} + +void mmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = mmOpen; + mgmtFp.closeFp = mmClose; + mgmtFp.startFp = mmStart; + mgmtFp.createMsgFp = mmProcessCreateReq; + mgmtFp.dropMsgFp = mmProcessDropReq; + mgmtFp.requiredFp = mmRequire; + + mmInitMsgHandles(pWrapper); + pWrapper->name = "mnode"; + pWrapper->fp = mgmtFp; +} + +int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + + int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey); + dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); + return code; +} + +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); +} diff --git a/source/dnode/mgmt/mnode/src/mmMsg.c b/source/dnode/mgmt/mnode/src/mmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..d04077baf8f0d26264800f318fca3f4e3b1b4841 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmMsg.c @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateMnodeReq createReq = {0}; + if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.replica <= 1 || createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create mnode since %s", terrstr()); + return -1; + } else { + return mmOpenFromMsg(pWrapper, &createReq); + } +} + +int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropMnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop mnode since %s", terrstr()); + return -1; + } else { + return mmDrop(pWrapper); + } +} + +int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDAlterMnodeReq alterReq = {0}; + if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (alterReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to alter mnode since %s", terrstr()); + return -1; + } else { + return mmAlter(pMgmt, &alterReq); + } +} + +void mmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by DNODE + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + + // Requests handled by MNODE + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + + // Requests handled by VNODE + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + +} diff --git a/source/dnode/mgmt/mnode/src/mmWorker.c b/source/dnode/mgmt/mnode/src/mmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..27489b45d0a960a285d6a5be0e72e266e2db17cd --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmWorker.c @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SMnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, will be processed in mnode queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + if (pMsg->rpcMsg.msgType != TDMT_DND_ALTER_MNODE) { + pMsg->pNode = pMgmt->pMnode; + code = mndProcessMsg(pMsg); + } else { + code = mmProcessAlterReq(pMgmt, pMsg); + } + + if (pRpc->msgType & 1U) { + if (pRpc->handle == NULL) return; + if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0) code = terrno; + SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; + dndSendRsp(pMgmt->pWrapper, &rsp); + } + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + +static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SMnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, will be processed in mnode queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + pMsg->pNode = pMgmt->pMnode; + code = mndProcessMsg(pMsg); + + if (pRpc->msgType & 1U) { + if (pRpc->handle == NULL) return; + if (code != 0) { + SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle}; + dndSendRsp(pMgmt->pWrapper, &rsp); + } + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + + +static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SNodeMsg *pMsg) { + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + return taosWriteQitem(pWorker->queue, pMsg); +} + +int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); +} + +int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); +} + +int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); +} + +int32_t mmProcessQueryMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); +} + +static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg == NULL) { + return -1; + } + + dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + pMsg->rpcMsg = *pRpc; + + int32_t code = taosWriteQitem(pWorker->queue, pMsg); + if (code != 0) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + return code; +} + +int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->writeWorker, pRpc); +} + +int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc); +} + +int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->queryWorker, pRpc); +} + + +int32_t mmStartWorker(SMnodeMgmt *pMgmt) { + SSingleWorkerCfg cfg = {.minNum = 0, .maxNum = 1, .name = "mnode-read", .fp = (FItem)mmProcessQueue, .param = pMgmt}; + SSingleWorkerCfg queryCfg = {.minNum = 0, .maxNum = 1, .name = "mnode-query", .fp = (FItem)mmProcessQueryQueue, .param = pMgmt}; + + if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) { + dError("failed to start mnode-query worker since %s", terrstr()); + return -1; + } + + if (tSingleWorkerInit(&pMgmt->readWorker, &cfg) != 0) { + dError("failed to start mnode-read worker since %s", terrstr()); + return -1; + } + + if (tSingleWorkerInit(&pMgmt->writeWorker, &cfg) != 0) { + dError("failed to start mnode-write worker since %s", terrstr()); + return -1; + } + + if (tSingleWorkerInit(&pMgmt->syncWorker, &cfg) != 0) { + dError("failed to start mnode sync-worker since %s", terrstr()); + return -1; + } + + dDebug("mnode workers are initialized"); + return 0; +} + +void mmStopWorker(SMnodeMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->readWorker); + tSingleWorkerCleanup(&pMgmt->queryWorker); + tSingleWorkerCleanup(&pMgmt->writeWorker); + tSingleWorkerCleanup(&pMgmt->syncWorker); + dDebug("mnode workers are closed"); +} diff --git a/source/dnode/mgmt/impl/inc/dndTransport.h b/source/dnode/mgmt/qnode/inc/qm.h similarity index 68% rename from source/dnode/mgmt/impl/inc/dndTransport.h rename to source/dnode/mgmt/qnode/inc/qm.h index e0ea21cba8e16f41bae17c6e5dccd0bb8558480e..e28ea3e9483f92c5405e6cf910330de973885e8c 100644 --- a/source/dnode/mgmt/impl/inc/dndTransport.h +++ b/source/dnode/mgmt/qnode/inc/qm.h @@ -13,21 +13,19 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_TRANSPORT_H_ -#define _TD_DND_TRANSPORT_H_ +#ifndef _TD_DND_QNODE_H_ +#define _TD_DND_QNODE_H_ + +#include "dnd.h" #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitTrans(SDnode *pDnode); -void dndCleanupTrans(SDnode *pDnode); -int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pRpcMsg); +void qmGetMgmtFp(SMgmtWrapper *pMgmt); #ifdef __cplusplus } #endif -#endif /*_TD_DND_TRANSPORT_H_*/ +#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qnode/inc/qmInt.h b/source/dnode/mgmt/qnode/inc/qmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..52d23a445c06714b80129119647824429aceb91a --- /dev/null +++ b/source/dnode/mgmt/qnode/inc/qmInt.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_QNODE_INT_H_ +#define _TD_DND_QNODE_INT_H_ + +#include "qm.h" +#include "qnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SQnodeMgmt { + SQnode *pQnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SSingleWorker queryWorker; + SSingleWorker fetchWorker; +} SQnodeMgmt; + +// qmInt.c +int32_t qmOpen(SMgmtWrapper *pWrapper); +int32_t qmDrop(SMgmtWrapper *pWrapper); + +// qmMsg.c +void qmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// qmWorker.c +int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t qmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype); + +int32_t qmStartWorker(SQnodeMgmt *pMgmt); +void qmStopWorker(SQnodeMgmt *pMgmt); +int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_QNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qnode/src/qmInt.c b/source/dnode/mgmt/qnode/src/qmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..c8cb7258c3873f9267244877caacddf0b6d6548b --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmInt.c @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.queueFps[QUERY_QUEUE] = qmPutMsgToQueryQueue; + msgCb.queueFps[FETCH_QUEUE] = qmPutMsgToFetchQueue; + msgCb.qsizeFp = qmGetQueueSize; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + pOption->msgCb = msgCb; +} + +static int32_t qmOpenImp(SQnodeMgmt *pMgmt) { + SQnodeOpt option = {0}; + qmInitOption(pMgmt, &option); + + pMgmt->pQnode = qndOpen(&option); + if (pMgmt->pQnode == NULL) { + dError("failed to open qnode since %s", terrstr()); + return -1; + } + + if (qmStartWorker(pMgmt) != 0) { + dError("failed to start qnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write qnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void qmCloseImp(SQnodeMgmt *pMgmt) { + if (pMgmt->pQnode != NULL) { + qmStopWorker(pMgmt); + qndClose(pMgmt->pQnode); + pMgmt->pQnode = NULL; + } +} + +int32_t qmDrop(SMgmtWrapper *pWrapper) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("qnode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop qnode since %s", terrstr()); + return -1; + } + + qmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + taosMemoryFree(pMgmt); + dInfo("qnode-mgmt is dropped"); + return 0; +} + +static void qmClose(SMgmtWrapper *pWrapper) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("qnode-mgmt start to cleanup"); + qmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + taosMemoryFree(pMgmt); + dInfo("qnode-mgmt is cleaned up"); +} + +int32_t qmOpen(SMgmtWrapper *pWrapper) { + dInfo("qnode-mgmt start to init"); + SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = qmOpenImp(pMgmt); + if (code != 0) { + dError("failed to init qnode-mgmt since %s", terrstr()); + qmClose(pWrapper); + } else { + dInfo("qnode-mgmt is initialized"); + } + + return code; +} + +void qmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = qmOpen; + mgmtFp.closeFp = qmClose; + mgmtFp.createMsgFp = qmProcessCreateReq; + mgmtFp.dropMsgFp = qmProcessDropReq; + mgmtFp.requiredFp = qmRequire; + + qmInitMsgHandles(pWrapper); + pWrapper->name = "qnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/qnode/src/qmMsg.c b/source/dnode/mgmt/qnode/src/qmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..ebe6477e8187e29bdb97181d2f3e886f968c2db3 --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmMsg.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateQnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create qnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return qmOpen(pWrapper); + } +} + +int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropQnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop qnode since %s", terrstr()); + return -1; + } else { + return qmDrop(pWrapper); + } +} + +void qmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by VNODE + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)qmProcessQueryMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)qmProcessQueryMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); +} diff --git a/source/dnode/mgmt/qnode/src/qmWorker.c b/source/dnode/mgmt/qnode/src/qmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..aa4da8279074485f29a2afdfbce2291e13d3933d --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmWorker.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; + dndSendRsp(pWrapper, &rsp); +} + +static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SQnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, will be processed in qnode-query queue", pMsg); + int32_t code = qndProcessQueryMsg(pMgmt->pQnode, &pMsg->rpcMsg); + if (code != 0) { + qmSendRsp(pMgmt->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static void qmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SQnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, will be processed in qnode-fetch queue", pMsg); + int32_t code = qndProcessFetchMsg(pMgmt->pQnode, &pMsg->rpcMsg); + if (code != 0) { + qmSendRsp(pMgmt->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static int32_t qmPutMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) { + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + return taosWriteQitem(pWorker->queue, pMsg); +} + +int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { return qmPutMsgToWorker(&pMgmt->queryWorker, pMsg); } + +int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { return qmPutMsgToWorker(&pMgmt->fetchWorker, pMsg); } + +static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg == NULL) { + return -1; + } + + dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + pMsg->rpcMsg = *pRpc; + + int32_t code = taosWriteQitem(pWorker->queue, pMsg); + if (code != 0) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + return code; +} + +int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + return qmPutRpcMsgToWorker(pMgmt, &pMgmt->queryWorker, pRpc); +} + +int32_t qmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + return qmPutRpcMsgToWorker(pMgmt, &pMgmt->fetchWorker, pRpc); +} + +int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { + int32_t size = -1; + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + switch (qtype) { + case QUERY_QUEUE: + size = taosQueueSize(pMgmt->queryWorker.queue); + break; + case FETCH_QUEUE: + size = taosQueueSize(pMgmt->fetchWorker.queue); + break; + default: + break; + } + + return size; +} + +int32_t qmStartWorker(SQnodeMgmt *pMgmt) { + int32_t maxFetchThreads = 4; + int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); + int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); + int32_t maxQueryThreads = minQueryThreads; + + SSingleWorkerCfg queryCfg = {.minNum = minQueryThreads, + .maxNum = maxQueryThreads, + .name = "qnode-query", + .fp = (FItem)qmProcessQueryQueue, + .param = pMgmt}; + + if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) { + dError("failed to start qnode-query worker since %s", terrstr()); + return -1; + } + + SSingleWorkerCfg fetchCfg = {.minNum = minFetchThreads, + .maxNum = maxFetchThreads, + .name = "qnode-fetch", + .fp = (FItem)qmProcessFetchQueue, + .param = pMgmt}; + + if (tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg) != 0) { + dError("failed to start qnode-fetch worker since %s", terrstr()); + return -1; + } + + dDebug("qnode workers are initialized"); + return 0; +} + +void qmStopWorker(SQnodeMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->queryWorker); + tSingleWorkerCleanup(&pMgmt->fetchWorker); + dDebug("qnode workers are closed"); +} diff --git a/source/dnode/mgmt/snode/inc/sm.h b/source/dnode/mgmt/snode/inc/sm.h new file mode 100644 index 0000000000000000000000000000000000000000..82a52e5d1fdf1577f705a6cebbe86f4f4ecebe27 --- /dev/null +++ b/source/dnode/mgmt/snode/inc/sm.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_SNODE_H_ +#define _TD_DND_SNODE_H_ + +#include "dnd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void smGetMgmtFp(SMgmtWrapper *pWrapper); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/snode/inc/smInt.h b/source/dnode/mgmt/snode/inc/smInt.h new file mode 100644 index 0000000000000000000000000000000000000000..f2b510483ce5fa3eb1343285406d2aae30bd7092 --- /dev/null +++ b/source/dnode/mgmt/snode/inc/smInt.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_SNODE_INT_H_ +#define _TD_DND_SNODE_INT_H_ + +#include "sm.h" +#include "snode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SSnodeMgmt { + SSnode *pSnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SRWLatch latch; + int8_t uniqueWorkerInUse; + SArray *uniqueWorkers; // SArray + SSingleWorker sharedWorker; +} SSnodeMgmt; + +// smInt.c +int32_t smOpen(SMgmtWrapper *pWrapper); +int32_t smDrop(SMgmtWrapper *pWrapper); + +// smMsg.c +void smInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// smWorker.c +int32_t smStartWorker(SSnodeMgmt *pMgmt); +void smStopWorker(SSnodeMgmt *pMgmt); +int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_SNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/snode/src/smInt.c b/source/dnode/mgmt/snode/src/smInt.c new file mode 100644 index 0000000000000000000000000000000000000000..351f7d656eedd61fee3faa555caa377e60df8974 --- /dev/null +++ b/source/dnode/mgmt/snode/src/smInt.c @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + pOption->msgCb = msgCb; +} + +static int32_t smOpenImp(SSnodeMgmt *pMgmt) { + SSnodeOpt option = {0}; + smInitOption(pMgmt, &option); + + pMgmt->pSnode = sndOpen(pMgmt->path, &option); + if (pMgmt->pSnode == NULL) { + dError("failed to open snode since %s", terrstr()); + return -1; + } + + if (smStartWorker(pMgmt) != 0) { + dError("failed to start snode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write snode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void smCloseImp(SSnodeMgmt *pMgmt) { + if (pMgmt->pSnode != NULL) { + smStopWorker(pMgmt); + sndClose(pMgmt->pSnode); + pMgmt->pSnode = NULL; + } +} + +int32_t smDrop(SMgmtWrapper *pWrapper) { + SSnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("snode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop snode since %s", terrstr()); + return -1; + } + + smCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + taosMemoryFree(pMgmt); + dInfo("snode-mgmt is dropped"); + return 0; +} + +static void smClose(SMgmtWrapper *pWrapper) { + SSnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("snode-mgmt start to cleanup"); + smCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + taosMemoryFree(pMgmt); + dInfo("snode-mgmt is cleaned up"); +} + +int32_t smOpen(SMgmtWrapper *pWrapper) { + dInfo("snode-mgmt start to init"); + SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = smOpenImp(pMgmt); + if (code != 0) { + dError("failed to init snode-mgmt since %s", terrstr()); + smClose(pWrapper); + } else { + dInfo("snode-mgmt is initialized"); + } + + return code; +} + +void smGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = smOpen; + mgmtFp.closeFp = smClose; + mgmtFp.createMsgFp = smProcessCreateReq; + mgmtFp.dropMsgFp = smProcessDropReq; + mgmtFp.requiredFp = smRequire; + + smInitMsgHandles(pWrapper); + pWrapper->name = "snode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/snode/src/smMsg.c b/source/dnode/mgmt/snode/src/smMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..aea1dded5679bacc9e6fd5fc2dcf01295acad99a --- /dev/null +++ b/source/dnode/mgmt/snode/src/smMsg.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateSnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create snode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return smOpen(pWrapper); + } +} + +int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropSnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop snode since %s", terrstr()); + return -1; + } else { + return smDrop(pWrapper); + } +} + +void smInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by SNODE + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg, VND_VGID); +} diff --git a/source/dnode/mgmt/snode/src/smWorker.c b/source/dnode/mgmt/snode/src/smWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..5913713ff3bb6ffb0d291a27cec85d6dabee32e8 --- /dev/null +++ b/source/dnode/mgmt/snode/src/smWorker.c @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SSnodeMgmt *pMgmt = pInfo->ahandle; + + for (int32_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + + dTrace("msg:%p, will be processed in snode unique queue", pMsg); + sndProcessUMsg(pMgmt->pSnode, &pMsg->rpcMsg); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } +} + +static void smProcessSharedQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SSnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, will be processed in snode shared queue", pMsg); + sndProcessSMsg(pMgmt->pSnode, &pMsg->rpcMsg); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +int32_t smStartWorker(SSnodeMgmt *pMgmt) { + pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(SMultiWorker *)); + if (pMgmt->uniqueWorkers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { + SMultiWorker *pUniqueWorker = taosMemoryMalloc(sizeof(SMultiWorker)); + if (pUniqueWorker == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SMultiWorkerCfg cfg = {.maxNum = 1, .name = "snode-unique", .fp = smProcessUniqueQueue, .param = pMgmt}; + + if (tMultiWorkerInit(pUniqueWorker, &cfg) != 0) { + dError("failed to start snode-unique worker since %s", terrstr()); + return -1; + } + if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + SSingleWorkerCfg cfg = {.minNum = SND_SHARED_THREAD_NUM, + .maxNum = SND_SHARED_THREAD_NUM, + .name = "snode-shared", + .fp = (FItem)smProcessSharedQueue, + .param = pMgmt}; + + if (tSingleWorkerInit(&pMgmt->sharedWorker, &cfg)) { + dError("failed to start snode shared-worker since %s", terrstr()); + return -1; + } + + dDebug("snode workers are initialized"); + return 0; +} + +void smStopWorker(SSnodeMgmt *pMgmt) { + for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { + SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, i); + tMultiWorkerCleanup(pWorker); + } + taosArrayDestroy(pMgmt->uniqueWorkers); + tSingleWorkerCleanup(&pMgmt->sharedWorker); + dDebug("snode workers are closed"); +} + +static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { + SMsgHead *pHead = pMsg->pCont; + pHead->vgId = htonl(pHead->vgId); + return pHead->vgId % SND_UNIQUE_THREAD_NUM; +} + +static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { + /*SMsgHead *pHead = pMsg->pCont;*/ + /*pHead->workerType = htonl(pHead->workerType);*/ + /*return pHead->workerType;*/ + return 0; +} + +int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0); + if (pWorker == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return taosWriteQitem(pWorker->queue, pMsg); +} + +int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg); + SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index); + if (pWorker == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return taosWriteQitem(pWorker->queue, pMsg); +} + +int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SSingleWorker *pWorker = &pMgmt->sharedWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return taosWriteQitem(pWorker->queue, pMsg); +} + +int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + int32_t workerType = smGetSWTypeFromMsg(&pMsg->rpcMsg); + if (workerType == SND_WORKER_TYPE__SHARED) { + return smProcessSharedMsg(pMgmt, pMsg); + } else { + return smProcessUniqueMsg(pMgmt, pMsg); + } +} diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/test/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/CMakeLists.txt rename to source/dnode/mgmt/test/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt b/source/dnode/mgmt/test/bnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/bnode/CMakeLists.txt rename to source/dnode/mgmt/test/bnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp b/source/dnode/mgmt/test/bnode/dbnode.cpp similarity index 88% rename from source/dnode/mgmt/impl/test/bnode/dbnode.cpp rename to source/dnode/mgmt/test/bnode/dbnode.cpp index 75f111587f2c8c8f827f12bff28639cbd3227681..ee81780921ccd09350d6256685a16c302fbfbd05 100644 --- a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp +++ b/source/dnode/mgmt/test/bnode/dbnode.cpp @@ -13,7 +13,10 @@ class DndTestBnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_snode", 9112); } + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_bnode", 9112); + taosMsleep(1100); + } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -36,7 +39,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +65,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -76,11 +79,11 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } -TEST_F(DndTestBnode, 01_Drop_Bnode) { +TEST_F(DndTestBnode, 02_Drop_Bnode) { { SDDropBnodeReq dropReq = {0}; dropReq.dnodeId = 2; @@ -91,7 +94,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -117,7 +120,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -132,7 +135,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { @@ -147,4 +150,4 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/impl/test/mnode/CMakeLists.txt b/source/dnode/mgmt/test/mnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/mnode/CMakeLists.txt rename to source/dnode/mgmt/test/mnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp similarity index 92% rename from source/dnode/mgmt/impl/test/mnode/dmnode.cpp rename to source/dnode/mgmt/test/mnode/dmnode.cpp index 8655bcb774fa1ea0125a3a8cc4755d38b201bb8d..4072eb90a82c3d1239e8d037bc705b95e0464da0 100644 --- a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -40,7 +40,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } { @@ -57,7 +57,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } { @@ -77,7 +77,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -96,7 +96,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -113,7 +113,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -145,7 +145,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -171,7 +171,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { @@ -188,7 +188,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt b/source/dnode/mgmt/test/qnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/qnode/CMakeLists.txt rename to source/dnode/mgmt/test/qnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp similarity index 91% rename from source/dnode/mgmt/impl/test/qnode/dqnode.cpp rename to source/dnode/mgmt/test/qnode/dqnode.cpp index 46c31539a2edd3bde58ba7780cc20132c8a31a0f..343814b15939b60836959674437da47833aa89ab 100644 --- a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -36,7 +36,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +62,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -92,7 +92,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -118,7 +118,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -133,7 +133,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/snode/CMakeLists.txt b/source/dnode/mgmt/test/snode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/snode/CMakeLists.txt rename to source/dnode/mgmt/test/snode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp similarity index 91% rename from source/dnode/mgmt/impl/test/snode/dsnode.cpp rename to source/dnode/mgmt/test/snode/dsnode.cpp index ea98dfdd9558bea777358a40504d451b8bc70872..8dfa437fd8596c2d9d76eef670c7b5cc2fd39411 100644 --- a/source/dnode/mgmt/impl/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -36,7 +36,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +62,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -92,7 +92,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -118,7 +118,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -133,7 +133,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/sut/CMakeLists.txt b/source/dnode/mgmt/test/sut/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/sut/CMakeLists.txt rename to source/dnode/mgmt/test/sut/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/sut/inc/client.h b/source/dnode/mgmt/test/sut/inc/client.h similarity index 100% rename from source/dnode/mgmt/impl/test/sut/inc/client.h rename to source/dnode/mgmt/test/sut/inc/client.h diff --git a/source/dnode/mgmt/impl/test/sut/inc/server.h b/source/dnode/mgmt/test/sut/inc/server.h similarity index 86% rename from source/dnode/mgmt/impl/test/sut/inc/server.h rename to source/dnode/mgmt/test/sut/inc/server.h index 99554a7aa78c2e3ffa57ab848a8ccc29c4c7281e..ad2d4a76e91cf9b6d0de787e407fcdd41e4282f7 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/server.h +++ b/source/dnode/mgmt/test/sut/inc/server.h @@ -24,11 +24,11 @@ class TestServer { bool DoStart(); private: - SDnodeObjCfg BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + SDnodeOpt BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); private: SDnode* pDnode; - pthread_t* threadId; + TdThread threadId; char path[PATH_MAX]; char fqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/test/sut/inc/sut.h similarity index 98% rename from source/dnode/mgmt/impl/test/sut/inc/sut.h rename to source/dnode/mgmt/test/sut/inc/sut.h index 304a370bcd4af4b5dda70b5131f9aadc95994ffb..f7e2831160519e18fa2bd8d1d0885922a41efdae 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/test/sut/inc/sut.h @@ -98,7 +98,7 @@ class Testbase { #define CheckBinaryByte(b, len) \ { \ - char* bytes = (char*)calloc(1, len); \ + char* bytes = (char*)taosMemoryCalloc(1, len); \ for (int32_t i = 0; i < len - 1; ++i) { \ bytes[i] = b; \ } \ diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/test/sut/src/client.cpp similarity index 96% rename from source/dnode/mgmt/impl/test/sut/src/client.cpp rename to source/dnode/mgmt/test/sut/src/client.cpp index f22bc9d27696a270c9913acc44ce97fe90795e6c..a1165d5bc9170345402483dc24985700fa65f535 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/test/sut/src/client.cpp @@ -25,9 +25,9 @@ static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { void TestClient::SetRpcRsp(SRpcMsg* rsp) { if (this->pRsp) { - free(this->pRsp); + taosMemoryFree(this->pRsp); } - this->pRsp = (SRpcMsg*)calloc(1, sizeof(SRpcMsg)); + this->pRsp = (SRpcMsg*)taosMemoryCalloc(1, sizeof(SRpcMsg)); this->pRsp->msgType = rsp->msgType; this->pRsp->code = rsp->code; this->pRsp->pCont = rsp->pCont; diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/test/sut/src/server.cpp similarity index 57% rename from source/dnode/mgmt/impl/test/sut/src/server.cpp rename to source/dnode/mgmt/test/sut/src/server.cpp index 985625b41c28d2518e13a7b01db022b4f276f890..c5379c6d174dce3f4dce5f1cba834e93a5549207 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/test/sut/src/server.cpp @@ -16,36 +16,37 @@ #include "sut.h" void* serverLoop(void* param) { - while (1) { - taosMsleep(100); - pthread_testcancel(); - } + SDnode* pDnode = (SDnode*)param; + dndRun(pDnode); + return NULL; } -SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SDnodeObjCfg cfg = {0}; - cfg.numOfSupportVnodes = 16; - cfg.serverPort = port; - strcpy(cfg.dataDir, path); - snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); - snprintf(cfg.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(cfg.firstEp, TSDB_EP_LEN, "%s", firstEp); - return cfg; +SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SDnodeOpt option = {0}; + option.numOfSupportVnodes = 16; + option.serverPort = port; + strcpy(option.dataDir, path); + snprintf(option.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(option.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); + snprintf(option.firstEp, TSDB_EP_LEN, "%s", firstEp); + return option; } bool TestServer::DoStart() { - SDnodeObjCfg cfg = BuildOption(path, fqdn, port, firstEp); + SDnodeOpt option = BuildOption(path, fqdn, port, firstEp); taosMkDir(path); - pDnode = dndCreate(&cfg); - if (pDnode != NULL) { + pDnode = dndCreate(&option); + if (pDnode == NULL) { return false; } - threadId = taosCreateThread(serverLoop, NULL); - if (threadId != NULL) { - return false; - } + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&threadId, &thAttr, serverLoop, pDnode); + taosThreadAttrDestroy(&thAttr); + taosMsleep(2100); return true; } @@ -67,10 +68,8 @@ bool TestServer::Start(const char* path, const char* fqdn, uint16_t port, const } void TestServer::Stop() { - if (threadId != NULL) { - taosDestoryThread(threadId); - threadId = NULL; - } + dndHandleEvent(pDnode, DND_EVENT_STOP); + taosThreadJoin(threadId, NULL); if (pDnode != NULL) { dndClose(pDnode); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp similarity index 98% rename from source/dnode/mgmt/impl/test/sut/src/sut.cpp rename to source/dnode/mgmt/test/sut/src/sut.cpp index 2e38ea75137f4120b4611db8c6f133164e05cee2..14197153b49bb2552d29409801d35ce40d8090c6 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -21,8 +21,8 @@ void Testbase::InitLog(const char* path) { mDebugFlag = 143; cDebugFlag = 0; jniDebugFlag = 0; - tmrDebugFlag = 143; - uDebugFlag = 143; + tmrDebugFlag = 135; + uDebugFlag = 135; rpcDebugFlag = 143; qDebugFlag = 0; wDebugFlag = 0; @@ -49,7 +49,6 @@ void Testbase::Init(const char* path, int16_t port) { InitLog("/tmp/td"); server.Start(path, fqdn, port, firstEp); client.Init("root", "taosdata", fqdn, port); - taosMsleep(1100); tFreeSTableMetaRsp(&metaRsp); showId = 0; diff --git a/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt b/source/dnode/mgmt/test/vnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/vnode/CMakeLists.txt rename to source/dnode/mgmt/test/vnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp similarity index 99% rename from source/dnode/mgmt/impl/test/vnode/vnode.cpp rename to source/dnode/mgmt/test/vnode/vnode.cpp index 4457faf7b1aa97215fa9433385e9e0438f8cd5da..e2de3fc33ae473b76cfea9c83ccfb4e03d2e8f82 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -158,6 +158,7 @@ TEST_F(DndTestVnode, 03_Create_Stb) { for (int i = 0; i < 1; ++i) { SVCreateTbReq req = {0}; req.ver = 0; + req.dbFName = (char*)"1.db1"; req.name = (char*)"stb1"; req.ttl = 0; req.keep = 0; @@ -229,6 +230,7 @@ TEST_F(DndTestVnode, 04_Alter_Stb) { for (int i = 0; i < 1; ++i) { SVCreateTbReq req = {0}; req.ver = 0; + req.dbFName = (char*)"1.db1"; req.name = (char*)"stb1"; req.ttl = 0; req.keep = 0; diff --git a/source/dnode/mgmt/impl/inc/dndQnode.h b/source/dnode/mgmt/vnode/inc/vm.h similarity index 53% rename from source/dnode/mgmt/impl/inc/dndQnode.h rename to source/dnode/mgmt/vnode/inc/vm.h index 2a25dca1c6dc324f3d98d425e5962df0a410aa13..60d9cfc3a175344014870a103078555c22cc569c 100644 --- a/source/dnode/mgmt/impl/inc/dndQnode.h +++ b/source/dnode/mgmt/vnode/inc/vm.h @@ -13,24 +13,34 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_QNODE_H_ -#define _TD_DND_QNODE_H_ +#ifndef _TD_DND_VNODES_H_ +#define _TD_DND_VNODES_H_ + +#include "dnd.h" #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitQnode(SDnode *pDnode); -void dndCleanupQnode(SDnode *pDnode); +typedef struct { + int32_t openVnodes; + int32_t totalVnodes; + int32_t masterNum; + int64_t numOfSelectReqs; + int64_t numOfInsertReqs; + int64_t numOfInsertSuccessReqs; + int64_t numOfBatchInsertReqs; + int64_t numOfBatchInsertSuccessReqs; +} SVnodesStat; + +void vmGetMgmtFp(SMgmtWrapper *pWrapper); -void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); #ifdef __cplusplus } #endif -#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file +#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/inc/vmInt.h b/source/dnode/mgmt/vnode/inc/vmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..197c606a0d582b28ea57755452d9336326f522ed --- /dev/null +++ b/source/dnode/mgmt/vnode/inc/vmInt.h @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_VNODES_INT_H_ +#define _TD_DND_VNODES_INT_H_ + +#include "sync.h" +#include "vm.h" +#include "vnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SVnodesMgmt { + SHashObj *hash; + SRWLatch latch; + SVnodesStat state; + STfs *pTfs; + SQWorkerPool queryPool; + SQWorkerPool fetchPool; + SWWorkerPool syncPool; + SWWorkerPool writePool; + SWWorkerPool mergePool; + const char *path; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + SSingleWorker mgmtWorker; +} SVnodesMgmt; + +typedef struct { + int32_t vgId; + int32_t vgVersion; + int8_t dropped; + uint64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; + char path[PATH_MAX + 20]; +} SWrapperCfg; + +typedef struct { + int32_t vgId; + int32_t refCount; + int32_t vgVersion; + int8_t dropped; + int8_t accessState; + uint64_t dbUid; + char *db; + char *path; + SVnode *pImpl; + STaosQueue *pWriteQ; + STaosQueue *pSyncQ; + STaosQueue *pApplyQ; + STaosQueue *pQueryQ; + STaosQueue *pFetchQ; + STaosQueue *pMergeQ; + SMgmtWrapper *pWrapper; +} SVnodeObj; + +typedef struct { + int32_t vnodeNum; + int32_t opened; + int32_t failed; + int32_t threadIndex; + TdThread thread; + SVnodesMgmt *pMgmt; + SWrapperCfg *pCfgs; +} SVnodeThread; + +// vmInt.c +SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId); +void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); +int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); +void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); + +// vmMsg.c +void vmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); + +// vmFile.c +int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); +int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt); +SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes); + +// vmWorker.c +int32_t vmStartWorker(SVnodesMgmt *pMgmt); +void vmStopWorker(SVnodesMgmt *pMgmt); +int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); +void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); + +int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype); + +int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessMergeMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_VNODES_INT_H_*/ diff --git a/source/dnode/mgmt/vnode/src/vmFile.c b/source/dnode/mgmt/vnode/src/vmFile.c new file mode 100644 index 0000000000000000000000000000000000000000..d4f906b482c8c2f21e2fa84ea96b91bb3ecf3348 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmFile.c @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes) { + taosRLockLatch(&pMgmt->latch); + + int32_t num = 0; + int32_t size = taosHashGetSize(pMgmt->hash); + SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *)); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + SVnodeObj *pVnode = *ppVnode; + if (pVnode && num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + pVnodes[num] = (*ppVnode); + num++; + pIter = taosHashIterate(pMgmt->hash, pIter); + } else { + taosHashCancelIterate(pMgmt->hash, pIter); + } + } + + taosRUnLockLatch(&pMgmt->latch); + *numOfVnodes = num; + + return pVnodes; +} + +int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 30000; + char *content = taosMemoryCalloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; + char file[PATH_MAX]; + SWrapperCfg *pCfgs = NULL; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_VNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_VNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_VNODE_OVER; + } + + cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); + if (!vnodes || vnodes->type != cJSON_Array) { + dError("failed to read %s since vnodes not found", file); + goto PRASE_VNODE_OVER; + } + + int32_t vnodesNum = cJSON_GetArraySize(vnodes); + if (vnodesNum > 0) { + pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg)); + if (pCfgs == NULL) { + dError("failed to read %s since out of memory", file); + goto PRASE_VNODE_OVER; + } + + for (int32_t i = 0; i < vnodesNum; ++i) { + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + SWrapperCfg *pCfg = &pCfgs[i]; + + cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); + if (!vgId || vgId->type != cJSON_Number) { + dError("failed to read %s since vgId not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgId = vgId->valueint; + snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); + + cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dropped = dropped->valueint; + + cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); + if (!vgVersion || vgVersion->type != cJSON_Number) { + dError("failed to read %s since vgVersion not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgVersion = vgVersion->valueint; + + cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); + if (!dbUid || dbUid->type != cJSON_String) { + dError("failed to read %s since dbUid not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dbUid = atoll(dbUid->valuestring); + + cJSON *db = cJSON_GetObjectItem(vnode, "db"); + if (!db || db->type != cJSON_String) { + dError("failed to read %s since db not found", file); + goto PRASE_VNODE_OVER; + } + tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN); + } + + *ppCfgs = pCfgs; + } + + *numOfVnodes = vnodesNum; + code = 0; + dInfo("succcessed to read file %s", file); + +PRASE_VNODE_OVER: + if (content != NULL) taosMemoryFree(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { + char file[PATH_MAX]; + char realfile[PATH_MAX]; + snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); + snprintf(realfile, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = vmGetVnodesFromHash(pMgmt, &numOfVnodes); + + int32_t len = 0; + int32_t maxLen = 65536; + char *content = taosMemoryCalloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + len += snprintf(content + len, maxLen - len, " {\n"); + len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); + len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); + if (i < numOfVnodes - 1) { + len += snprintf(content + len, maxLen - len, " },\n"); + } else { + len += snprintf(content + len, maxLen - len, " }\n"); + } + } + len += snprintf(content + len, maxLen - len, " ]\n"); + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + taosMemoryFree(content); + terrno = 0; + + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + vmReleaseVnode(pMgmt, pVnode); + } + + if (pVnodes != NULL) { + taosMemoryFree(pVnodes); + } + + dDebug("successed to write %s", realfile); + return taosRenameFile(file, realfile); +} \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/src/vmInt.c b/source/dnode/mgmt/vnode/src/vmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..464e789e468bc496e8680649b63267db8768ac3e --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmInt.c @@ -0,0 +1,415 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { + SVnodeObj *pVnode = NULL; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + if (pVnode == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + } else { + refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + if (pVnode != NULL) { + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + } + + return pVnode; +} + +void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + if (pVnode == NULL) return; + + taosRLockLatch(&pMgmt->latch); + int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + taosRUnLockLatch(&pMgmt->latch); + dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); +} + +int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { + SVnodeObj *pVnode = taosMemoryCalloc(1, sizeof(SVnodeObj)); + if (pVnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pVnode->vgId = pCfg->vgId; + pVnode->refCount = 0; + pVnode->dropped = 0; + pVnode->accessState = TSDB_VN_ALL_ACCCESS; + pVnode->pWrapper = pMgmt->pWrapper; + pVnode->pImpl = pImpl; + pVnode->vgVersion = pCfg->vgVersion; + pVnode->dbUid = pCfg->dbUid; + pVnode->db = tstrdup(pCfg->db); + pVnode->path = tstrdup(pCfg->path); + + if (pVnode->path == NULL || pVnode->db == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (vmAllocQueue(pMgmt, pVnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + taosWLockLatch(&pMgmt->latch); + int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); + taosWUnLockLatch(&pMgmt->latch); + + if (code != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return code; +} + +void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + taosWLockLatch(&pMgmt->latch); + taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); + taosWUnLockLatch(&pMgmt->latch); + + vmReleaseVnode(pMgmt, pVnode); + while (pVnode->refCount > 0) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + + vmFreeQueue(pMgmt, pVnode); + vnodeClose(pVnode->pImpl); + pVnode->pImpl = NULL; + + dDebug("vgId:%d, vnode is closed", pVnode->vgId); + + if (pVnode->dropped) { + dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped); + vnodeDestroy(pVnode->path); + } + + taosMemoryFree(pVnode->path); + taosMemoryFree(pVnode->db); + taosMemoryFree(pVnode); +} + +static void *vmOpenVnodeFunc(void *param) { + SVnodeThread *pThread = param; + SVnodesMgmt *pMgmt = pThread->pMgmt; + SDnode *pDnode = pMgmt->pDnode; + + dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + setThreadName("open-vnodes"); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + SWrapperCfg *pCfg = &pThread->pCfgs[v]; + + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, + pMgmt->state.openVnodes, pMgmt->state.totalVnodes); + dndReportStartup(pDnode, "open-vnodes", stepDesc); + + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; + msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; + msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; + msgCb.qsizeFp = vmGetQueueSize; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; + SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); + if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); + pThread->failed++; + } else { + vmOpenVnode(pMgmt, pCfg, pImpl); + dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); + pThread->opened++; + } + + atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); + } + + dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + taosInitRWLatch(&pMgmt->latch); + + pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMgmt->hash == NULL) { + dError("failed to init vnode hash"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SWrapperCfg *pCfgs = NULL; + int32_t numOfVnodes = 0; + if (vmGetVnodesFromFile(pMgmt, &pCfgs, &numOfVnodes) != 0) { + dInfo("failed to get vnode list from disk since %s", terrstr()); + return -1; + } + + pMgmt->state.totalVnodes = numOfVnodes; + +#if 0 + int32_t threadNum = tsNumOfCores; +#else + int32_t threadNum = 1; +#endif + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + + SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].pMgmt = pMgmt; + threads[t].pCfgs = taosMemoryCalloc(vnodesPerThread, sizeof(SWrapperCfg)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SVnodeThread *pThread = &threads[t]; + pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; + } + + dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pThread->thread, &thAttr, vmOpenVnodeFunc, pThread) != 0) { + dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + } + + taosThreadAttrDestroy(&thAttr); + } + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + taosThreadJoin(pThread->thread, NULL); + } + taosMemoryFree(pThread->pCfgs); + } + taosMemoryFree(threads); + taosMemoryFree(pCfgs); + + if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { + dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); + return -1; + } else { + dInfo("total vnodes:%d open successfully", pMgmt->state.totalVnodes); + return 0; + } +} + +static void vmCloseVnodes(SVnodesMgmt *pMgmt) { + dInfo("start to close all vnodes"); + + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = vmGetVnodesFromHash(pMgmt, &numOfVnodes); + + for (int32_t i = 0; i < numOfVnodes; ++i) { + vmCloseVnode(pMgmt, pVnodes[i]); + } + + if (pVnodes != NULL) { + taosMemoryFree(pVnodes); + } + + if (pMgmt->hash != NULL) { + taosHashCleanup(pMgmt->hash); + pMgmt->hash = NULL; + } + + dInfo("total vnodes:%d are all closed", numOfVnodes); +} + +static void vmCleanup(SMgmtWrapper *pWrapper) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("vnode-mgmt start to cleanup"); + vmCloseVnodes(pMgmt); + vmStopWorker(pMgmt); + vnodeCleanup(); + // walCleanUp(); + taosMemoryFree(pMgmt); + pWrapper->pMgmt = NULL; + dInfo("vnode-mgmt is cleaned up"); +} + +static int32_t vmInit(SMgmtWrapper *pWrapper) { + SDnode *pDnode = pWrapper->pDnode; + SVnodesMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodesMgmt)); + int32_t code = -1; + + dInfo("vnode-mgmt start to init"); + if (pMgmt == NULL) goto _OVER; + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + taosInitRWLatch(&pMgmt->latch); + + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, pDnode->dataDir, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + SDiskCfg *pDisks = pDnode->pDisks; + int32_t numOfDisks = pDnode->numOfDisks; + if (numOfDisks <= 0 || pDisks == NULL) { + pDisks = &dCfg; + numOfDisks = 1; + } + + pMgmt->pTfs = tfsOpen(pDisks, numOfDisks); + if (pMgmt->pTfs == NULL) { + dError("failed to init tfs since %s", terrstr()); + goto _OVER; + } + + if (walInit() != 0) { + dError("failed to init wal since %s", terrstr()); + goto _OVER; + } + + if (vnodeInit() != 0) { + dError("failed to init vnode since %s", terrstr()); + goto _OVER; + } + + if (vmStartWorker(pMgmt) != 0) { + dError("failed to init workers since %s", terrstr()) goto _OVER; + } + + if (vmOpenVnodes(pMgmt) != 0) { + dError("failed to open vnode since %s", terrstr()); + return -1; + } + + code = 0; + +_OVER: + if (code == 0) { + pWrapper->pMgmt = pMgmt; + dInfo("vnodes-mgmt is initialized"); + } else { + dError("failed to init vnodes-mgmt since %s", terrstr()); + vmCleanup(pWrapper); + } + + return 0; +} + +static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { + SDnode *pDnode = pWrapper->pDnode; + *required = pDnode->numOfSupportVnodes > 0; + return 0; +} + +void vmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = vmInit; + mgmtFp.closeFp = vmCleanup; + mgmtFp.requiredFp = vmRequire; + + vmInitMsgHandles(pWrapper); + pWrapper->name = "vnode"; + pWrapper->fp = mgmtFp; +} + +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return -1; + + return tfsGetMonitorInfo(pMgmt->pTfs, pInfo); +} + +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + SVnodesStat *pStat = &pMgmt->state; + pInfo->req_select = pStat->numOfSelectReqs; + pInfo->req_insert = pStat->numOfInsertReqs; + pInfo->req_insert_success = pStat->numOfInsertSuccessReqs; + pInfo->req_insert_batch = pStat->numOfBatchInsertReqs; + pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs; + pInfo->errors = tsNumOfErrorLogs; + pInfo->vnodes_num = pStat->totalVnodes; + pInfo->masters = pStat->masterNum; +} + +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + SVnodesStat *pStat = &pMgmt->state; + int32_t totalVnodes = 0; + int32_t masterNum = 0; + int64_t numOfSelectReqs = 0; + int64_t numOfInsertReqs = 0; + int64_t numOfInsertSuccessReqs = 0; + int64_t numOfBatchInsertReqs = 0; + int64_t numOfBatchInsertSuccessReqs = 0; + + taosRLockLatch(&pMgmt->latch); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + if (ppVnode == NULL || *ppVnode == NULL) continue; + + SVnodeObj *pVnode = *ppVnode; + SVnodeLoad vload = {0}; + vnodeGetLoad(pVnode->pImpl, &vload); + taosArrayPush(pLoads, &vload); + + numOfSelectReqs += vload.numOfSelectReqs; + numOfInsertReqs += vload.numOfInsertReqs; + numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs; + numOfBatchInsertReqs += vload.numOfBatchInsertReqs; + numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs; + totalVnodes++; + if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++; + + pIter = taosHashIterate(pMgmt->hash, pIter); + } + + taosRUnLockLatch(&pMgmt->latch); + + pStat->totalVnodes = totalVnodes; + pStat->masterNum = masterNum; + pStat->numOfSelectReqs = numOfSelectReqs; + pStat->numOfInsertReqs = numOfInsertReqs; + pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs; + pStat->numOfBatchInsertReqs = numOfBatchInsertReqs; + pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; +} \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..97d829571fe4438e46b59c4a677d296b789d9b81 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { + pCfg->vgId = pCreate->vgId; + pCfg->wsize = pCreate->cacheBlockSize; + pCfg->ssize = pCreate->cacheBlockSize; + pCfg->lsize = pCreate->cacheBlockSize; + pCfg->isHeapAllocator = true; + pCfg->ttl = 4; + pCfg->keep = pCreate->daysToKeep0; + pCfg->streamMode = pCreate->streamMode; + pCfg->isWeak = true; + pCfg->tsdbCfg.keep = pCreate->daysToKeep0; + pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; + pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; + pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; + pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; + pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; + pCfg->walCfg.level = pCreate->walLevel; + pCfg->walCfg.retentionPeriod = 10; + pCfg->walCfg.retentionSize = 128; + pCfg->walCfg.rollPeriod = 128; + pCfg->walCfg.segSize = 128; + pCfg->walCfg.vgId = pCreate->vgId; + pCfg->hashBegin = pCreate->hashBegin; + pCfg->hashEnd = pCreate->hashEnd; + pCfg->hashMethod = pCreate->hashMethod; +} + +static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { + memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN); + pCfg->dbUid = pCreate->dbUid; + pCfg->dropped = 0; + snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCreate->vgId); + pCfg->vgId = pCreate->vgId; + pCfg->vgVersion = pCreate->vgVersion; +} + +int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SCreateVnodeReq createReq = {0}; + if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dDebug("vgId:%d, create vnode req is received", createReq.vgId); + + SVnodeCfg vnodeCfg = {0}; + vmGenerateVnodeCfg(&createReq, &vnodeCfg); + + SWrapperCfg wrapperCfg = {0}; + vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); + + if (createReq.dnodeId != pMgmt->pDnode->dnodeId) { + terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; + dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + return -1; + } + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); + if (pVnode != NULL) { + dDebug("vgId:%d, already exist", createReq.vgId); + vmReleaseVnode(pMgmt, pVnode); + terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; + return -1; + } + + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; + msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; + msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; + msgCb.qsizeFp = vmGetQueueSize; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + + vnodeCfg.msgCb = msgCb; + vnodeCfg.pTfs = pMgmt->pTfs; + vnodeCfg.dbId = wrapperCfg.dbUid; + SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); + if (pImpl == NULL) { + dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + return -1; + } + + int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl); + if (code != 0) { + dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + code = vmWriteVnodesToFile(pMgmt); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + return 0; +} + +int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SAlterVnodeReq alterReq = {0}; + if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dDebug("vgId:%d, alter vnode req is received", alterReq.vgId); + + SVnodeCfg vnodeCfg = {0}; + vmGenerateVnodeCfg(&alterReq, &vnodeCfg); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, alterReq.vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); + return -1; + } + + if (alterReq.vgVersion == pVnode->vgVersion) { + vmReleaseVnode(pMgmt, pVnode); + dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId); + return 0; + } + + if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { + dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + int32_t oldVersion = pVnode->vgVersion; + pVnode->vgVersion = alterReq.vgVersion; + int32_t code = vmWriteVnodesToFile(pMgmt); + if (code != 0) { + pVnode->vgVersion = oldVersion; + } + + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SDropVnodeReq dropReq = {0}; + if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t vgId = dropReq.vgId; + dDebug("vgId:%d, drop vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); + terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; + return -1; + } + + pVnode->dropped = 1; + if (vmWriteVnodesToFile(pMgmt) != 0) { + pVnode->dropped = 0; + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmCloseVnode(pMgmt, pVnode); + vmWriteVnodesToFile(pMgmt); + + return 0; +} + +int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SSyncVnodeReq syncReq = {0}; + tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq); + + int32_t vgId = syncReq.vgId; + dDebug("vgId:%d, sync vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); + return -1; + } + + if (vnodeSync(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmReleaseVnode(pMgmt, pVnode); + return 0; +} + +int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SCompactVnodeReq compatcReq = {0}; + tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq); + + int32_t vgId = compatcReq.vgId; + dDebug("vgId:%d, compact vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); + return -1; + } + + if (vnodeCompact(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmReleaseVnode(pMgmt, pVnode); + return 0; +} + +void vmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by VNODE + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); +} diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..e97d6e7f116c1c0f084cfa772a56367d2704bee5 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -0,0 +1,397 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; + dndSendRsp(pWrapper, &rsp); +} + +static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SVnodesMgmt *pMgmt = pInfo->ahandle; + + int32_t code = -1; + tmsg_t msgType = pMsg->rpcMsg.msgType; + dTrace("msg:%p, will be processed in vnode-mgmt queue", pMsg); + + switch (msgType) { + case TDMT_DND_CREATE_VNODE: + code = vmProcessCreateVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_ALTER_VNODE: + code = vmProcessAlterVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_DROP_VNODE: + code = vmProcessDropVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_SYNC_VNODE: + code = vmProcessSyncVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_COMPACT_VNODE: + code = vmProcessCompactVnodeReq(pMgmt, pMsg); + break; + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + dError("msg:%p, not processed in vnode-mgmt queue", pMsg); + } + + if (msgType & 1u) { + if (code != 0 && terrno != 0) code = terrno; + vmSendRsp(pMgmt->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static void vmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SVnodeObj *pVnode = pInfo->ahandle; + + dTrace("msg:%p, will be processed in vnode-query queue", pMsg); + int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, &pMsg->rpcMsg); + if (code != 0) { + vmSendRsp(pVnode->pWrapper, pMsg, code); + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } +} + +static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SVnodeObj *pVnode = pInfo->ahandle; + + dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg); + int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg); + if (code != 0) { + vmSendRsp(pVnode->pWrapper, pMsg, code); + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } +} + +static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SVnodeObj *pVnode = pInfo->ahandle; + + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); + if (pArray == NULL) { + dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr()); + return; + } + + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + + dTrace("msg:%p, will be processed in vnode-write queue", pMsg); + if (taosArrayPush(pArray, &pMsg) == NULL) { + dTrace("msg:%p, failed to process since %s", pMsg, terrstr()); + vmSendRsp(pVnode->pWrapper, pMsg, TSDB_CODE_OUT_OF_MEMORY); + } + } + + vnodeProcessWMsgs(pVnode->pImpl, pArray); + + numOfMsgs = taosArrayGetSize(pArray); + for (int32_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + SRpcMsg *pRpc = &pMsg->rpcMsg; + SRpcMsg *pRsp = NULL; + + int32_t code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp); + if (pRsp != NULL) { + pRsp->ahandle = pRpc->ahandle; + dndSendRsp(pVnode->pWrapper, pRsp); + taosMemoryFree(pRsp); + } else { + if (code != 0 && terrno != 0) code = terrno; + vmSendRsp(pVnode->pWrapper, pMsg, code); + } + } + + for (int32_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } + + taosArrayDestroy(pArray); +} + +static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SVnodeObj *pVnode = pInfo->ahandle; + SNodeMsg *pMsg = NULL; + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + + // todo + SRpcMsg *pRsp = NULL; + (void)vnodeApplyWMsg(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + } +} + +static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SVnodeObj *pVnode = pInfo->ahandle; + SNodeMsg *pMsg = NULL; + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + + // todo + SRpcMsg *pRsp = NULL; + (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + } +} + +static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + SMsgHead *pHead = pRpc->pCont; + pHead->contLen = ntohl(pHead->contLen); + pHead->vgId = ntohl(pHead->vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); + if (pVnode == NULL) { + dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr()); + return -1; + } + + switch (qtype) { + case QUERY_QUEUE: + dTrace("msg:%p, will be written into vnode-query queue", pMsg); + code = taosWriteQitem(pVnode->pQueryQ, pMsg); + break; + case FETCH_QUEUE: + dTrace("msg:%p, will be written into vnode-fetch queue", pMsg); + code = taosWriteQitem(pVnode->pFetchQ, pMsg); + break; + case WRITE_QUEUE: + dTrace("msg:%p, will be written into vnode-write queue", pMsg); + code = taosWriteQitem(pVnode->pWriteQ, pMsg); + break; + case SYNC_QUEUE: + dTrace("msg:%p, will be written into vnode-sync queue", pMsg); + code = taosWriteQitem(pVnode->pSyncQ, pMsg); + break; + case MERGE_QUEUE: + dTrace("msg:%p, will be written into vnode-merge queue", pMsg); + code = taosWriteQitem(pVnode->pMergeQ, pMsg); + break; + default: + terrno = TSDB_CODE_INVALID_PARA; + break; + } + + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); } + +int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, WRITE_QUEUE); } + +int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, QUERY_QUEUE); } + +int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); } + +int32_t vmProcessMergeMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, MERGE_QUEUE); } + +int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SSingleWorker *pWorker = &pMgmt->mgmtWorker; + dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); + return taosWriteQitem(pWorker->queue, pMsg); +} + +static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + int32_t code = -1; + SMsgHead *pHead = pRpc->pCont; + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); + if (pVnode == NULL) return -1; + + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg != NULL) { + dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + pMsg->rpcMsg = *pRpc; + switch (qtype) { + case QUERY_QUEUE: + dTrace("msg:%p, will be put into vnode-query queue", pMsg); + code = taosWriteQitem(pVnode->pQueryQ, pMsg); + break; + case FETCH_QUEUE: + dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); + code = taosWriteQitem(pVnode->pFetchQ, pMsg); + break; + case APPLY_QUEUE: + dTrace("msg:%p, will be put into vnode-apply queue", pMsg); + code = taosWriteQitem(pVnode->pApplyQ, pMsg); + break; + case MERGE_QUEUE: + dTrace("msg:%p, will be put into vnode-merge queue", pMsg); + code = taosWriteQitem(pVnode->pMergeQ, pMsg); + break; + default: + terrno = TSDB_CODE_INVALID_PARA; + break; + } + } + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, QUERY_QUEUE); +} + +int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, FETCH_QUEUE); +} + +int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, APPLY_QUEUE); +} + +int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE); +} + +int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { + int32_t size = -1; + SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId); + if (pVnode != NULL) { + switch (qtype) { + case QUERY_QUEUE: + size = taosQueueSize(pVnode->pQueryQ); + break; + case FETCH_QUEUE: + size = taosQueueSize(pVnode->pFetchQ); + break; + case WRITE_QUEUE: + size = taosQueueSize(pVnode->pWriteQ); + break; + case SYNC_QUEUE: + size = taosQueueSize(pVnode->pSyncQ); + break; + case APPLY_QUEUE: + size = taosQueueSize(pVnode->pApplyQ); + break; + case MERGE_QUEUE: + size = taosQueueSize(pVnode->pMergeQ); + break; + default: + break; + } + } + vmReleaseVnode(pWrapper->pMgmt, pVnode); + return size; +} + +int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); + pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); + pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeMsg); + pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); + pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); + pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); + + if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || + pVnode->pQueryQ == NULL || pVnode->pMergeQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + dDebug("vgId:%d, vnode queue is alloced", pVnode->vgId); + return 0; +} + +void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); + tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tWWorkerFreeQueue(&pMgmt->mergePool, pVnode->pMergeQ); + tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); + pVnode->pWriteQ = NULL; + pVnode->pApplyQ = NULL; + pVnode->pSyncQ = NULL; + pVnode->pFetchQ = NULL; + pVnode->pQueryQ = NULL; + pVnode->pMergeQ = NULL; + dDebug("vgId:%d, vnode queue is freed", pVnode->vgId); +} + +int32_t vmStartWorker(SVnodesMgmt *pMgmt) { + int32_t maxFetchThreads = 4; + int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); + int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); + int32_t maxQueryThreads = minQueryThreads; + int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); + int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); + int32_t maxMergeThreads = 1; + + SQWorkerPool *pQPool = &pMgmt->queryPool; + pQPool->name = "vnode-query"; + pQPool->min = minQueryThreads; + pQPool->max = maxQueryThreads; + if (tQWorkerInit(pQPool) != 0) return -1; + + SQWorkerPool *pFPool = &pMgmt->fetchPool; + pFPool->name = "vnode-fetch"; + pFPool->min = minFetchThreads; + pFPool->max = maxFetchThreads; + if (tQWorkerInit(pFPool) != 0) return -1; + + SWWorkerPool *pWPool = &pMgmt->writePool; + pWPool->name = "vnode-write"; + pWPool->max = maxWriteThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + + pWPool = &pMgmt->syncPool; + pWPool->name = "vnode-sync"; + pWPool->max = maxSyncThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + + pWPool = &pMgmt->mergePool; + pWPool->name = "vnode-merge"; + pWPool->max = maxMergeThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + + SSingleWorkerCfg cfg = { + .minNum = 1, .maxNum = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) { + dError("failed to start vnode-mgmt worker since %s", terrstr()); + return -1; + } + + dDebug("vnode workers are initialized"); + return 0; +} + +void vmStopWorker(SVnodesMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->mgmtWorker); + tQWorkerCleanup(&pMgmt->fetchPool); + tQWorkerCleanup(&pMgmt->queryPool); + tWWorkerCleanup(&pMgmt->writePool); + tWWorkerCleanup(&pMgmt->syncPool); + tWWorkerCleanup(&pMgmt->mergePool); + dDebug("vnode workers are closed"); +} diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 514bba19f48cf1625c9d4b74c8aabb6a296ace15..e91c851be7de7fff34e8853b0be5a38896a1a553 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mnode scheduler sdb wal transport cjson sync monitor + mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser ) if(${BUILD_TEST}) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 8ea9cc141f8d673efc8e33cb734a6854f2d8bbe9..eb7ac5b3539ac8cbd94495583293c227671c768c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -26,6 +26,7 @@ #include "tlog.h" #include "tmsg.h" #include "trpc.h" +#include "tstream.h" #include "ttimer.h" #include "mnode.h" @@ -103,6 +104,8 @@ typedef enum { TRN_TYPE_CREATE_STB = 4001, TRN_TYPE_ALTER_STB = 4002, TRN_TYPE_DROP_STB = 4003, + TRN_TYPE_CREATE_SMA = 4004, + TRN_TYPE_DROP_SMA = 4005, TRN_TYPE_STB_SCOPE_END, } ETrnType; @@ -265,6 +268,8 @@ typedef struct { int8_t update; int8_t cacheLastRow; int8_t streamMode; + int32_t numOfRetensions; + SArray* pRetensions; } SDbCfg; typedef struct { @@ -305,6 +310,31 @@ typedef struct { SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; + int32_t sqlLen; + int32_t astLen; + char* expr; + char* tagsFilter; + char* sql; + char* ast; +} SSmaObj; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; @@ -314,12 +344,19 @@ typedef struct { int64_t dbUid; int32_t version; int32_t nextColId; + float xFilesFactor; + int32_t aggregationMethod; + int32_t delay; + int32_t ttl; int32_t numOfColumns; int32_t numOfTags; + int32_t numOfSmas; + int32_t commentLen; SSchema* pColumns; SSchema* pTags; + SSchema* pSmas; + char* comment; SRWLatch lock; - char comment[TSDB_STB_COMMENT_LEN]; } SStbObj; typedef struct { @@ -405,7 +442,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) { if (pConsumerEp) { - tfree(pConsumerEp->qmsg); + taosMemoryFreeClear(pConsumerEp->qmsg); } } @@ -474,7 +511,7 @@ typedef struct { } SMqSubscribeObj; static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { - SMqSubscribeObj* pSub = calloc(1, sizeof(SMqSubscribeObj)); + SMqSubscribeObj* pSub = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); if (pSub == NULL) { return NULL; } @@ -501,10 +538,10 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { return pSub; _err: - tfree(pSub->consumers); - tfree(pSub->lostConsumers); - tfree(pSub->unassignedVg); - tfree(pSub); + taosMemoryFreeClear(pSub->consumers); + taosMemoryFreeClear(pSub->lostConsumers); + taosMemoryFreeClear(pSub->unassignedVg); + taosMemoryFreeClear(pSub); return NULL; } @@ -684,6 +721,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; + char outputSTbName[TSDB_TABLE_FNAME_LEN]; int64_t createTime; int64_t updateTime; int64_t uid; @@ -692,27 +730,20 @@ typedef struct { int32_t vgNum; SRWLatch lock; int8_t status; + int8_t sourceType; + int8_t sinkType; // int32_t sqlLen; + int32_t sinkVgId; // 0 for automatic char* sql; char* logicalPlan; char* physicalPlan; - SArray* tasks; // SArray> + SArray* tasks; // SArray> + SArray* ColAlias; // SArray } SStreamObj; int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj); -typedef struct SMnodeMsg { - char user[TSDB_USER_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int32_t acctId; - SMnode* pMnode; - int64_t createdTime; - SRpcMsg rpcMsg; - int32_t contLen; - void* pCont; -} SMnodeMsg; - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 0fe24cd7575162a2ecb1bfe41e77ed6df49e2060..f89e9d8fe0536f4848bd5798ce3b4c4fe806e1f6 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -38,11 +38,11 @@ extern "C" { #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} -typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); +typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); -typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +typedef int32_t (*ShowMetaFp)(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct SMnodeLoad { @@ -59,6 +59,8 @@ typedef struct SMnodeLoad { int64_t compStorage; } SMnodeLoad; +typedef struct SQWorkerMgmt SQHandle; + typedef struct { const char *name; MndInitFp initFp; @@ -110,8 +112,9 @@ typedef struct SMnode { char *path; int64_t checkTime; SSdb *pSdb; - SDnode *pDnode; + SMgmtWrapper *pWrapper; SArray *pSteps; + SQHandle *pQuery; SShowMgmt showMgmt; SProfileMgmt profileMgmt; STelemMgmt telemMgmt; @@ -119,21 +122,12 @@ typedef struct SMnode { SHashObj *infosMeta; SGrantInfo grant; MndMsgFp msgFp[TDMT_MAX]; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; - PutReqToMWriteQFp putReqToMWriteQFp; - PutReqToMReadQFp putReqToMReadQFp; + SMsgCb msgCb; } SMnode; -int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg); -int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg); -void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg); void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); - -uint64_t mndGenerateUid(char *name, int32_t len); - -void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +int64_t mndGenerateUid(char *name, int32_t len); +void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mnode/impl/inc/mndQuery.h similarity index 70% rename from source/dnode/mgmt/daemon/inc/dmnInt.h rename to source/dnode/mnode/impl/inc/mndQuery.h index 8a571352f021ab53f3405e11114257b7ecbd4fd8..7fab80de77dc06f37c7b36403efc7b4efb6306ec 100644 --- a/source/dnode/mgmt/daemon/inc/dmnInt.h +++ b/source/dnode/mnode/impl/inc/mndQuery.h @@ -1,4 +1,3 @@ - /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -14,28 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_DMN_INT_H_ -#define _TD_DMN_INT_H_ +#ifndef _TD_MND_QUERY_H_ +#define _TD_MND_QUERY_H_ -#include "tconfig.h" -#include "dnode.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tlog.h" -#include "version.h" +#include "mndInt.h" #ifdef __cplusplus extern "C" { #endif -SDnodeObjCfg dmnGetObjCfg(); +int32_t mndInitQuery(SMnode *pMnode); +void mndCleanupQuery(SMnode *pMnode); -void dmnDumpCfg(); -void dmnPrintVersion(); -void dmnGenerateGrant(); #ifdef __cplusplus } #endif -#endif /*_TD_DMN_INT_H_*/ +#endif /*_TD_MND_QUERY_H_*/ diff --git a/source/dnode/mgmt/impl/inc/dndWorker.h b/source/dnode/mnode/impl/inc/mndSma.h similarity index 63% rename from source/dnode/mgmt/impl/inc/dndWorker.h rename to source/dnode/mnode/impl/inc/mndSma.h index 9c037d91c72f02e451dfbf379dc6de4b9ce6a8fc..4a80f619d3026c82abeb2c2e9c777073dbf2fa6a 100644 --- a/source/dnode/mgmt/impl/inc/dndWorker.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -13,21 +13,22 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_WORKER_H_ -#define _TD_DND_WORKER_H_ +#ifndef _TD_MND_SMA_H_ +#define _TD_MND_SMA_H_ + +#include "mndInt.h" #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, - int32_t maxNum, void *queueFp); -void dndCleanupWorker(SDnodeWorker *pWorker); -int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen); +int32_t mndInitSma(SMnode *pMnode); +void mndCleanupSma(SMnode *pMnode); +SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); +void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); #ifdef __cplusplus } #endif -#endif /*_TD_DND_WORKER_H_*/ \ No newline at end of file +#endif /*_TD_MND_SMA_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndSnode.h b/source/dnode/mnode/impl/inc/mndSnode.h index 8d64879605aa0ef6562ca94b88733111619bfe61..180f18a6ddc68a5cd93987714c3d2b42349a529f 100644 --- a/source/dnode/mnode/impl/inc/mndSnode.h +++ b/source/dnode/mnode/impl/inc/mndSnode.h @@ -24,6 +24,7 @@ extern "C" { int32_t mndInitSnode(SMnode *pMnode); void mndCleanupSnode(SMnode *pMnode); +SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index b1145b0c6bf598542661f2d2f556b5e2a9f4dc60..b5d22cb7a5ba931ccbf5c5986ec41d7cbf5e6e5e 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -31,6 +31,8 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); +int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index a0218eaf65807a571de7271b55a4ca4ccacf891a..5c1b0991be803e765637c8f36b3b84cdb1fc7056 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -47,7 +47,7 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); -void mndTransProcessRsp(SMnodeMsg *pRsp); +void mndTransProcessRsp(SNodeMsg *pRsp); void mndTransPullup(SMnode *pMnode); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 85e9a15bd422737ab29f05a431e7f9267cb5d096..f42829eddfffddabf55f96ad7ed830831f28378e 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -28,7 +28,7 @@ SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); -SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup); +SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index aa87eb43a18a84e04bef9056d3fde8991daa6d2a..68c8c6d12744b79433b5f00a3bfe3f97dab0495b 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -26,9 +26,9 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw); static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew); -static int32_t mndProcessCreateAcctReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterAcctReq(SMnodeMsg *pReq); -static int32_t mndProcessDropAcctReq(SMnodeMsg *pReq); +static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq); +static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq); +static int32_t mndProcessDropAcctReq(SNodeMsg *pReq); int32_t mndInitAcct(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_ACCT, @@ -158,7 +158,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { ACCT_DECODE_OVER: if (terrno != 0) { mError("acct:%s, failed to decode from raw:%p since %s", pAcct->acct, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -185,19 +185,19 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { return 0; } -static int32_t mndProcessCreateAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } -static int32_t mndProcessAlterAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } -static int32_t mndProcessDropAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessDropAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index ab8b9350b099e01d646b7fb9a43976f10fe93ad3..20dd7f1378ff857dd21c63cb6472668ce4ff2556 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -17,7 +17,7 @@ #include "mndAuth.h" #include "mndUser.h" -static int32_t mndProcessAuthReq(SMnodeMsg *pReq); +static int32_t mndProcessAuthReq(SNodeMsg *pReq); int32_t mndInitAuth(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq); @@ -45,7 +45,7 @@ int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, cha return 0; } -static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { +static int32_t mndProcessAuthReq(SNodeMsg *pReq) { SAuthReq authReq = {0}; if (tDeserializeSAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -56,15 +56,15 @@ static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { memcpy(authRsp.user, authReq.user, TSDB_USER_LEN); int32_t code = - mndRetriveAuth(pReq->pMnode, authRsp.user, &authRsp.spi, &authRsp.encrypt, authRsp.secret, authRsp.ckey); + mndRetriveAuth(pReq->pNode, authRsp.user, &authRsp.spi, &authRsp.encrypt, authRsp.secret, authRsp.ckey); mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->user, authRsp.spi, authRsp.encrypt, authRsp.user); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp); void *pRsp = rpcMallocCont(contLen); tSerializeSAuthReq(pRsp, contLen, &authRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; return code; } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index a9921354644c4837305767d0a2f1e2ef568b2654..c4868d980230c64ddbdfe05fff78566767f0d9fc 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -29,12 +29,12 @@ static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); -static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateBnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropBnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateBnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetBnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); int32_t mndInitBnode(SMnode *pMnode) { @@ -126,7 +126,7 @@ static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw) { BNODE_DECODE_OVER: if (terrno != 0) { mError("bnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -191,7 +191,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -203,10 +203,10 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -218,7 +218,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -230,17 +230,17 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } return 0; } -static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { +static int32_t mndCreateBnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { int32_t code = -1; SBnodeObj bnodeObj = {0}; @@ -266,8 +266,8 @@ CREATE_BNODE_OVER: return code; } -static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateBnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SBnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -341,7 +341,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -353,17 +353,17 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } return 0; } -static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pReq, SBnodeObj *pObj) { +static int32_t mndDropBnode(SMnode *pMnode, SNodeMsg *pReq, SBnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_BNODE, &pReq->rpcMsg); @@ -382,8 +382,8 @@ DROP_BNODE_OVER: return code; } -static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropBnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SBnodeObj *pObj = NULL; @@ -430,18 +430,18 @@ DROP_BNODE_OVER: return code; } -static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateBnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetBnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -480,8 +480,8 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 3410a386da2d80bafcb35db5f2d97e987f753eb3..dde7e1fe8fd946c0f9632dcb7273b89cbe246757 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -26,8 +26,8 @@ static int32_t mndClusterActionInsert(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOldCluster, SClusterObj *pNewCluster); static int32_t mndCreateDefaultCluster(SMnode *pMnode); -static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetClusterMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter); int32_t mndInitCluster(SMnode *pMnode) { @@ -116,7 +116,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { CLUSTER_DECODE_OVER: if (terrno != 0) { mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -163,7 +163,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } -static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { +static int32_t mndGetClusterMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchemas; @@ -201,8 +201,8 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4a9d23aa156852e7c6ed59c5ee194ad119ded677..8eceea3d06b43b66bf3ef0ba482c16e6d008170f 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -34,9 +34,9 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); -static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessConsumerMetaMsg(SNodeMsg *pMsg); +static int32_t mndGetConsumerMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConsumer(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); int32_t mndInitConsumer(SMnode *pMnode) { @@ -54,7 +54,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) { - SMqConsumerObj *pConsumer = calloc(1, sizeof(SMqConsumerObj)); + SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -79,7 +79,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto CM_ENCODE_OVER; void *abuf = buf; @@ -94,7 +94,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_SUCCESS; CM_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != 0) { mError("consumer:%" PRId64 ", failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -126,7 +126,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); - buf = malloc(len); + buf = taosMemoryMalloc(len); if (buf == NULL) goto CM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); @@ -138,10 +138,10 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; CM_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6625c7f104bb108b9f8e032cad947f8bc47b5e99..05c7d79a1a1d6424e8b44021918b57e713a34956 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -31,14 +31,14 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew); -static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq); -static int32_t mndProcessDropDbReq(SMnodeMsg *pReq); -static int32_t mndProcessUseDbReq(SMnodeMsg *pReq); -static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq); -static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq); -static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateDbReq(SNodeMsg *pReq); +static int32_t mndProcessAlterDbReq(SNodeMsg *pReq); +static int32_t mndProcessDropDbReq(SNodeMsg *pReq); +static int32_t mndProcessUseDbReq(SNodeMsg *pReq); +static int32_t mndProcessSyncDbReq(SNodeMsg *pReq); +static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); +static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); int32_t mndInitDb(SMnode *pMnode) { @@ -100,6 +100,15 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.quorum, DB_ENCODE_OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update, DB_ENCODE_OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, DB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, DB_ENCODE_OVER) + for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); + SDB_SET_INT32(pRaw, dataPos, pRetension->freq, DB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pRetension->keep, DB_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, DB_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, DB_ENCODE_OVER) + } + SDB_SET_RESERVE(pRaw, dataPos, TSDB_DB_RESERVE_SIZE, DB_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, DB_ENCODE_OVER) @@ -161,6 +170,22 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.quorum, DB_DECODE_OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.update, DB_DECODE_OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, DB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, DB_DECODE_OVER) + if (pDb->cfg.numOfRetensions > 0) { + pDb->cfg.pRetensions = taosArrayInit(pDb->cfg.numOfRetensions, sizeof(SRetention)); + if (pDb->cfg.pRetensions == NULL) goto DB_DECODE_OVER; + for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { + SRetention retension = {0}; + SDB_GET_INT32(pRaw, dataPos, &retension.freq, DB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &retension.keep, DB_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, &retension.freqUnit, DB_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, &retension.keepUnit, DB_DECODE_OVER) + if (taosArrayPush(pDb->cfg.pRetensions, &retension) == NULL) { + goto DB_DECODE_OVER; + } + } + } + SDB_GET_RESERVE(pRaw, dataPos, TSDB_DB_RESERVE_SIZE, DB_DECODE_OVER) terrno = 0; @@ -168,7 +193,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { DB_DECODE_OVER: if (terrno != 0) { mError("db:%s, failed to decode from raw:%p since %s", pDb->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -183,6 +208,7 @@ static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { mTrace("db:%s, perform delete action, row:%p", pDb->name, pDb); + taosArrayDestroy(pDb->cfg.pRetensions); return 0; } @@ -344,7 +370,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -375,7 +401,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -384,7 +410,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { +static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; memcpy(dbObj.name, pCreate->db, TSDB_DB_FNAME_LEN); memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); @@ -417,6 +443,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pReq, SCreateDbReq *pCreat .streamMode = pCreate->streamMode, }; + dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; + dbObj.cfg.pRetensions = pCreate->pRetensions; + pCreate = NULL; + mndSetDefaultDbCfg(&dbObj.cfg); if (mndCheckDbName(dbObj.name, pUser) != 0) { @@ -453,13 +483,13 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pReq, SCreateDbReq *pCreat code = 0; CREATE_DB_OVER: - free(pVgroups); + taosMemoryFree(pVgroups); mndTransDrop(pTrans); return code; } -static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -505,6 +535,7 @@ CREATE_DB_OVER: mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); + tFreeSCreateDbReq(&createReq); return code; } @@ -591,7 +622,7 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_DND_ALTER_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -622,7 +653,7 @@ static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndUpdateDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); if (pTrans == NULL) goto UPDATE_DB_OVER; @@ -642,8 +673,8 @@ UPDATE_DB_OVER: return code; } -static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -771,7 +802,7 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -802,7 +833,33 @@ static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { +static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bool useRpcMalloc) { + SDropDbRsp dropRsp = {0}; + if (pDb != NULL) { + memcpy(dropRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + dropRsp.uid = pDb->uid; + } + + int32_t rspLen = tSerializeSDropDbRsp(NULL, 0, &dropRsp); + void *pRsp = NULL; + if (useRpcMalloc) { + pRsp = rpcMallocCont(rspLen); + } else { + pRsp = taosMemoryMalloc(rspLen); + } + + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSDropDbRsp(pRsp, rspLen, &dropRsp); + *pRspLen = rspLen; + *ppRsp = pRsp; + return 0; +} + +static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_DB, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_DB_OVER; @@ -814,18 +871,9 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; - SDropDbRsp dropRsp = {0}; - memcpy(dropRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - dropRsp.uid = pDb->uid; - - int32_t rspLen = tSerializeSDropDbRsp(NULL, 0, &dropRsp); - void *pRsp = malloc(rspLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto DROP_DB_OVER; - } - tSerializeSDropDbRsp(pRsp, rspLen, &dropRsp); - + int32_t rspLen = 0; + void *pRsp = NULL; + if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto DROP_DB_OVER; mndTransSetRpcRsp(pTrans, pRsp, rspLen); if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_DB_OVER; @@ -837,8 +885,8 @@ DROP_DB_OVER: return code; } -static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -854,7 +902,7 @@ static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { pDb = mndAcquireDb(pMnode, dropReq.db); if (pDb == NULL) { if (dropReq.ignoreNotExists) { - code = 0; + code = mndBuildDropDbRsp(pDb, &pReq->rspLen, &pReq->pRsp, true); goto DROP_DB_OVER; } else { terrno = TSDB_CODE_MND_DB_NOT_EXIST; @@ -913,12 +961,12 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; - while (vindex < pDb->cfg.numOfVgroups) { + while (true) { SVgObj *pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; - if (pVgroup->dbUid == pDb->uid) { + if (NULL == pDb || pVgroup->dbUid == pDb->uid) { SVgroupInfo vgInfo = {0}; vgInfo.vgId = pVgroup->vgId; vgInfo.hashBegin = pVgroup->hashBegin; @@ -943,13 +991,17 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { } sdbRelease(pSdb, pVgroup); + + if (pDb && (vindex >= pDb->cfg.numOfVgroups)) { + break; + } } sdbCancelFetch(pSdb, pIter); } -static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -964,7 +1016,27 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { char *p = strchr(usedbReq.db, '.'); if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); - code = 0; + static int32_t vgVersion = 1; + if (usedbReq.vgVersion < vgVersion) { + usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } + + mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); + usedbRsp.vgVersion = vgVersion++; + + if (taosArrayGetSize(usedbRsp.pVgroupInfos) <= 0) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + } + } else { + usedbRsp.vgVersion = usedbReq.vgVersion; + code = 0; + } + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + + // no jump, need to construct rsp } else { pDb = mndAcquireDb(pMnode, usedbReq.db); if (pDb == NULL) { @@ -1017,8 +1089,8 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; USE_DB_OVER: if (code != 0) { @@ -1086,7 +1158,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, } int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeSUseDbBatchRsp(&batchUseRsp); @@ -1101,8 +1173,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } -static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessSyncDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -1142,8 +1214,8 @@ SYNC_DB_OVER: return code; } -static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCompactDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -1183,8 +1255,8 @@ SYNC_DB_OVER: return code; } -static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -1324,123 +1396,149 @@ char *mnGetDbStr(char *src) { return pos; } -static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - SDbObj *pDb = NULL; - char *pWrite; +static char* getDataPosition(char* pData, SShowObj* pShow, int32_t cols, int32_t rows, int32_t capacityOfRow) { + return pData + pShow->offset[cols] * capacityOfRow + pShow->bytes[cols] * rows; +} + +static void dumpDbInfoToPayload(char* data, SDbObj* pDb, SShowObj* pShow, int32_t rows, int32_t rowCapacity, int64_t numOfTables) { int32_t cols = 0; - while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); - if (pShow->pIter == NULL) break; + char* pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + char *name = mnGetDbStr(pDb->name); + if (name != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); + } else { + STR_TO_VARSTR(pWrite, "NULL"); + } + cols++; - cols = 0; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int64_t *)pWrite = pDb->createdTime; + cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char *name = mnGetDbStr(pDb->name); - if (name != NULL) { - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); - } else { - STR_TO_VARSTR(pWrite, "NULL"); - } - cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int16_t *)pWrite = pDb->cfg.numOfVgroups; + cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pDb->createdTime; - cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int64_t *)pWrite = numOfTables; + cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDb->cfg.numOfVgroups; - cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int16_t *)pWrite = pDb->cfg.replications; + cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = 0; // todo - cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int16_t *)pWrite = pDb->cfg.quorum; + cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDb->cfg.replications; - cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int16_t *)pWrite = pDb->cfg.daysPerFile; + cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDb->cfg.quorum; - cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + char tmp[128] = {0}; + if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { + sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0); + } else { + sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2); + } + STR_WITH_SIZE_TO_VARSTR(pWrite, tmp, strlen(tmp)); + cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDb->cfg.daysPerFile; - cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; + cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char tmp[128] = {0}; - if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { - sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0); - } else { - sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2); - } - STR_WITH_SIZE_TO_VARSTR(pWrite, tmp, strlen(tmp)); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.totalBlocks; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.minRows; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxRows; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int8_t *)pWrite = pDb->cfg.walLevel; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.fsyncPeriod; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int8_t *)pWrite = pDb->cfg.compression; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int8_t *)pWrite = pDb->cfg.cacheLastRow; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char *prec = NULL; - switch (pDb->cfg.precision) { - case TSDB_TIME_PRECISION_MILLI: - prec = TSDB_TIME_PRECISION_MILLI_STR; - break; - case TSDB_TIME_PRECISION_MICRO: - prec = TSDB_TIME_PRECISION_MICRO_STR; - break; - case TSDB_TIME_PRECISION_NANO: - prec = TSDB_TIME_PRECISION_NANO_STR; - break; - default: - prec = "none"; - break; - } - STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); - cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int32_t *)pWrite = pDb->cfg.totalBlocks; + cols++; -// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; -// *(int8_t *)pWrite = pDb->cfg.update; -// cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int32_t *)pWrite = pDb->cfg.minRows; + cols++; + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int32_t *)pWrite = pDb->cfg.maxRows; + cols++; + + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int8_t *)pWrite = pDb->cfg.walLevel; + cols++; + + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int32_t *)pWrite = pDb->cfg.fsyncPeriod; + cols++; + + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int8_t *)pWrite = pDb->cfg.compression; + cols++; + + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + *(int8_t *)pWrite = pDb->cfg.cacheLastRow; + cols++; + + pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); + char *prec = NULL; + switch (pDb->cfg.precision) { + case TSDB_TIME_PRECISION_MILLI: + prec = TSDB_TIME_PRECISION_MILLI_STR; + break; + case TSDB_TIME_PRECISION_MICRO: + prec = TSDB_TIME_PRECISION_MICRO_STR; + break; + case TSDB_TIME_PRECISION_NANO: + prec = TSDB_TIME_PRECISION_NANO_STR; + break; + default: + prec = "none"; + break; + } + STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); + cols++; + +// pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); +// *(int8_t *)pWrite = pDb->cfg.update; +} + +static void setInformationSchemaDbCfg(SDbObj* pDbObj) { + ASSERT(pDbObj != NULL); + strncpy(pDbObj->name, TSDB_INFORMATION_SCHEMA_DB, tListLen(pDbObj->name)); + + pDbObj->createdTime = 0; + pDbObj->cfg.numOfVgroups = 0; + pDbObj->cfg.quorum = 1; + pDbObj->cfg.replications = 1; + pDbObj->cfg.update = 1; + pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; +} + +static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rowsCapacity) { + SMnode *pMnode = pReq->pNode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SDbObj *pDb = NULL; + + while (numOfRows < rowsCapacity) { + pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); + if (pShow->pIter == NULL) { + break; + } + + dumpDbInfoToPayload(data, pDb, pShow, numOfRows, rowsCapacity, 0); numOfRows++; sdbRelease(pSdb, pDb); } - mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + // Append the information_schema database into the result. + if (numOfRows < rowsCapacity) { + SDbObj dummyISDb = {0}; + setInformationSchemaDbCfg(&dummyISDb); + dumpDbInfoToPayload(data, &dummyISDb, pShow, numOfRows, rowsCapacity, 14); + numOfRows += 1; + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rowsCapacity, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index f81dead3259c3e43d0287829dfcd33be6ba2fad1..6fa926d548a1955cdb396b914101c1a5b7cbb113 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -16,6 +16,8 @@ #include "mndDef.h" int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { + int32_t sz = 0; + int32_t outputNameSz = 0; if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1; if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; @@ -29,19 +31,27 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; // TODO encode tasks if (pObj->tasks) { - int32_t sz = taosArrayGetSize(pObj->tasks); - tEncodeI32(pEncoder, sz); - for (int32_t i = 0; i < sz; i++) { - SArray *pArray = taosArrayGet(pObj->tasks, i); - int32_t innerSz = taosArrayGetSize(pArray); - tEncodeI32(pEncoder, innerSz); - for (int32_t j = 0; j < innerSz; j++) { - SStreamTask *pTask = taosArrayGet(pArray, j); - tEncodeSStreamTask(pEncoder, pTask); - } + sz = taosArrayGetSize(pObj->tasks); + } + if (tEncodeI32(pEncoder, sz) < 0) return -1; + + for (int32_t i = 0; i < sz; i++) { + SArray *pArray = taosArrayGet(pObj->tasks, i); + int32_t innerSz = taosArrayGetSize(pArray); + if (tEncodeI32(pEncoder, innerSz) < 0) return -1; + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask *pTask = taosArrayGet(pArray, j); + if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1; } - } else { - tEncodeI32(pEncoder, 0); + } + + if (pObj->ColAlias != NULL) { + outputNameSz = taosArrayGetSize(pObj->ColAlias); + } + if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1; + for (int32_t i = 0; i < outputNameSz; i++) { + char *name = taosArrayGetP(pObj->ColAlias, i); + if (tEncodeCStr(pEncoder, name) < 0) return -1; } return pEncoder->pos; } @@ -58,6 +68,7 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; + pObj->tasks = NULL; int32_t sz; if (tDecodeI32(pDecoder, &sz) < 0) return -1; if (sz != 0) { @@ -73,8 +84,19 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { } taosArrayPush(pObj->tasks, pArray); } - } else { - pObj->tasks = NULL; + } + int32_t outputNameSz; + if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1; + if (outputNameSz != 0) { + pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *)); + if (pObj->ColAlias == NULL) { + return -1; + } + } + for (int32_t i = 0; i < outputNameSz; i++) { + char *name; + if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1; + taosArrayPush(pObj->ColAlias, &name); } return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e231717e61de7ea717cdda2d4b686610bf1dd632..55414124606b6f526300d94d2575e14ecbb5332f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -49,17 +49,17 @@ static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew); -static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessStatusReq(SMnodeMsg *pReq); - -static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessStatusReq(SNodeMsg *pReq); + +static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); -static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetDnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); int32_t mndInitDnode(SMnode *pMnode) { @@ -164,7 +164,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { DNODE_DECODE_OVER: if (terrno != 0) { mError("dnode:%d, failed to decode from raw:%p since %s", pDnode->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -296,8 +296,8 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return 0; } -static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessStatusReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SStatusReq statusReq = {0}; SDnodeObj *pDnode = NULL; int32_t code = -1; @@ -424,8 +424,8 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { tSerializeSStatusRsp(pHead, contLen, &statusRsp); taosArrayDestroy(statusRsp.pDnodeEps); - pReq->contLen = contLen; - pReq->pCont = pHead; + pReq->rspLen = contLen; + pReq->pRsp = pHead; } pDnode->lastAccessTime = curMs; @@ -437,7 +437,7 @@ PROCESS_STATUS_MSG_OVER: return code; } -static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pReq, SCreateDnodeReq *pCreate) { +static int32_t mndCreateDnode(SMnode *pMnode, SNodeMsg *pReq, SCreateDnodeReq *pCreate) { SDnodeObj dnodeObj = {0}; dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE); dnodeObj.createdTime = taosGetTimestampMs(); @@ -471,8 +471,8 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pReq, SCreateDnodeReq * return 0; } -static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDnodeObj *pDnode = NULL; @@ -521,7 +521,7 @@ CREATE_DNODE_OVER: return code; } -static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) { +static int32_t mndDropDnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_DNODE, &pReq->rpcMsg); if (pTrans == NULL) { mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); @@ -547,8 +547,8 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) return 0; } -static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDnodeObj *pDnode = NULL; @@ -596,8 +596,8 @@ DROP_DNODE_OVER: return code; } -static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SMCfgDnodeReq cfgReq = {0}; if (tDeserializeSMCfgDnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { @@ -623,16 +623,16 @@ static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { .msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .ahandle = pReq->rpcMsg.ahandle}; mInfo("dnode:%d, app:%p config:%s req send to dnode", cfgReq.dnodeId, rpcMsg.ahandle, cfgReq.config); - mndSendReqToDnode(pMnode, &epSet, &rpcMsg); + tmsgSendReq(&pMnode->msgCb, &epSet, &rpcMsg); return 0; } -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) { mInfo("app:%p config rsp from dnode", pRsp->rpcMsg.ahandle); } -static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { +static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchemas; @@ -663,8 +663,8 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t totalRows = 0; int32_t numOfRows = 0; char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; @@ -709,8 +709,8 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {} -static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetDnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -752,7 +752,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; + pShow->bytes[cols] = 256 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "offline_reason"); pSchema[cols].bytes = pShow->bytes[cols]; @@ -773,8 +773,8 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b2daf848c53262f45fc18196398ba84ff4fe1d38..be4198f0d62c64896e28ac52e3b52c04e71ca031 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -29,13 +29,13 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew); -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate); -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc); -static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq); -static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq); -static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq); -static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCreate); +static int32_t mndDropFunc(SMnode *pMnode, SNodeMsg *pReq, SFuncObj *pFunc); +static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq); +static int32_t mndProcessDropFuncReq(SNodeMsg *pReq); +static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq); +static int32_t mndGetFuncMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); int32_t mndInitFunc(SMnode *pMnode) { @@ -127,8 +127,8 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER) - pFunc->pComment = calloc(1, pFunc->commentSize); - pFunc->pCode = calloc(1, pFunc->codeSize); + pFunc->pComment = taosMemoryCalloc(1, pFunc->commentSize); + pFunc->pCode = taosMemoryCalloc(1, pFunc->codeSize); if (pFunc->pComment == NULL || pFunc->pCode == NULL) { goto FUNC_DECODE_OVER; } @@ -142,7 +142,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { FUNC_DECODE_OVER: if (terrno != 0) { mError("func:%s, failed to decode from raw:%p since %s", pFunc->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -157,8 +157,8 @@ static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) { static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { mTrace("func:%s, perform delete action, row:%p", pFunc->name, pFunc); - tfree(pFunc->pCode); - tfree(pFunc->pComment); + taosMemoryFreeClear(pFunc->pCode); + taosMemoryFreeClear(pFunc->pComment); return 0; } @@ -181,7 +181,7 @@ static void mndReleaseFunc(SMnode *pMnode, SFuncObj *pFunc) { sdbRelease(pSdb, pFunc); } -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate) { +static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCreate) { int32_t code = -1; STrans *pTrans = NULL; @@ -196,8 +196,8 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pC func.signature = pCreate->signature; func.commentSize = pCreate->commentSize; func.codeSize = pCreate->codeSize; - func.pComment = malloc(func.commentSize); - func.pCode = malloc(func.codeSize); + func.pComment = taosMemoryMalloc(func.commentSize); + func.pCode = taosMemoryMalloc(func.codeSize); if (func.pCode == NULL || func.pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto CREATE_FUNC_OVER; @@ -228,13 +228,13 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pC code = 0; CREATE_FUNC_OVER: - free(func.pCode); - free(func.pComment); + taosMemoryFree(func.pCode); + taosMemoryFree(func.pComment); mndTransDrop(pTrans); return code; } -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc) { +static int32_t mndDropFunc(SMnode *pMnode, SNodeMsg *pReq, SFuncObj *pFunc) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_FUNC, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_FUNC_OVER; @@ -262,8 +262,8 @@ DROP_FUNC_OVER: return code; } -static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SFuncObj *pFunc = NULL; @@ -339,8 +339,8 @@ CREATE_FUNC_OVER: return code; } -static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SFuncObj *pFunc = NULL; @@ -394,8 +394,8 @@ DROP_FUNC_OVER: return code; } -static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncRsp retrieveRsp = {0}; @@ -451,8 +451,8 @@ static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; code = 0; @@ -463,8 +463,8 @@ RETRIEVE_FUNC_OVER: return code; } -static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetFuncMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -545,8 +545,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le return tDataTypes[type].name; } -static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SFuncObj *pFunc = NULL; diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 2c391e93e8ba0d6d0fd9950cfd3139215a043aed..c9b69bc1615ee78e6238c06b2c592df6f77a02a5 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -16,85 +16,93 @@ #define _DEFAULT_SOURCE #include "mndInfoSchema.h" -static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "vnodes", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "cores", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, +#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) + +//!!!! Note: only APPEND columns in below tables, NO insert !!!! +static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "offline_reason", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, }; static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, }; static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; -static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "vgroups", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "replica", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "quorum", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "keep", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "wallevel", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wallevel", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "comp", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "cachelast", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, +// {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update }; static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, }; -static const SInfosTableSchema userIdxSchema[] = {{.name = "table_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, +static const SInfosTableSchema userIdxSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "index_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, }; -static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +static const SInfosTableSchema userStbsSchema[] = {{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT}, }; -static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, +static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "dest_table", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY}, }; -static const SInfosTableSchema userTblsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +static const SInfosTableSchema userTblsSchema[] = { + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "tid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY}, {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, @@ -107,13 +115,15 @@ static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; -static const SInfosTableSchema userUsersSchema[] = {{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "privilege", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, +static const SInfosTableSchema userUsersSchema[] = {{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, }; -static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +static const SInfosTableSchema vgroupsSchema[] = {{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, @@ -122,13 +132,15 @@ static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", . {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)}, {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, - {TSDB_INS_TABLE_USER_DATABASE, userDBSchema, tListLen(userDBSchema)}, + {TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)}, {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, @@ -139,15 +151,14 @@ static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, }; - +//connection/application/ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) { - SSchema *schema = calloc(colNum, sizeof(SSchema)); + SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); if (NULL == schema) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - for (int32_t i = 0; i < colNum; ++i) { strcpy(schema[i].name, pSrc[i].name); @@ -157,7 +168,6 @@ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, S } *pDst = schema; - return TSDB_CODE_SUCCESS; } @@ -165,7 +175,7 @@ int32_t mndInsInitMeta(SHashObj *hash) { STableMetaRsp meta = {0}; strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB); - meta.tableType = TSDB_NORMAL_TABLE; + meta.tableType = TSDB_SYSTEM_TABLE; meta.sversion = 1; meta.tversion = 1; @@ -201,7 +211,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * *pRsp = *meta; - pRsp->pSchemas = calloc(meta->numOfColumns, sizeof(SSchema)); + pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema)); if (pRsp->pSchemas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pRsp->pSchemas = NULL; @@ -232,7 +242,7 @@ void mndCleanupInfos(SMnode *pMnode) { while (pIter) { STableMetaRsp *meta = (STableMetaRsp *)pIter; - tfree(meta->pSchemas); + taosMemoryFreeClear(meta->pSchemas); pIter = taosHashIterate(pMnode->infosMeta, pIter); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 9a6297a0f47f1293afc416aff67a071ab2cdab85..c408fea636b03b87e915995ad01a66981eb2a1cb 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -30,13 +30,13 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw); static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj); static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj); static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew); -static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveMnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateMnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropMnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateMnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessAlterMnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropMnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetMnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); int32_t mndInitMnode(SMnode *pMnode) { @@ -180,7 +180,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { MNODE_DECODE_OVER: if (terrno != 0) { mError("mnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -313,17 +313,17 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno createReq.dnodeId = pMObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_ALTER_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; @@ -338,16 +338,16 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno createReq.dnodeId = pObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -355,7 +355,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno return 0; } -static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { +static int32_t mndCreateMnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { int32_t code = -1; SMnodeObj mnodeObj = {0}; @@ -380,8 +380,8 @@ CREATE_MNODE_OVER: return code; } -static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateMnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SMnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -483,17 +483,17 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode alterReq.dnodeId = pMObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_ALTER_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; @@ -510,16 +510,16 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = pObj->id; int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -527,7 +527,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode return 0; } -static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pReq, SMnodeObj *pObj) { +static int32_t mndDropMnode(SMnode *pMnode, SNodeMsg *pReq, SMnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_MNODE, &pReq->rpcMsg); @@ -547,8 +547,8 @@ DROP_MNODE_OVER: return code; } -static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropMnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SMnodeObj *pObj = NULL; @@ -595,23 +595,23 @@ DROP_MNODE_OVER: return code; } -static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessAlterMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetMnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -663,8 +663,8 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveMnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; @@ -692,11 +692,11 @@ static int32_t mndRetrieveMnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pObj->createdTime; + *(int64_t *)pWrite = pObj->roleTime; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pObj->roleTime; + *(int64_t *)pWrite = pObj->createdTime; cols++; numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index ac9e99ebd43769559c47868275d84da2c4066001..22dad48772b2024cb2bb96243de070300aec4e0a 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -32,7 +32,7 @@ static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset); static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset); static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset); -static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq); +static int32_t mndProcessCommitOffsetReq(SNodeMsg *pReq); int32_t mndInitOffset(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_OFFSET, @@ -59,7 +59,7 @@ SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size); if (pRaw == NULL) goto OFFSET_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto OFFSET_ENCODE_OVER; void *abuf = buf; @@ -74,7 +74,7 @@ SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { terrno = TSDB_CODE_SUCCESS; OFFSET_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -107,7 +107,7 @@ SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto OFFSET_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER); @@ -119,10 +119,10 @@ SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; OFFSET_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -152,10 +152,10 @@ int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicNam return 0; } -static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) { +static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) { char key[TSDB_PARTITION_KEY_LEN]; - SMnode *pMnode = pMsg->pMnode; + SMnode *pMnode = pMsg->pNode; char *msgStr = pMsg->rpcMsg.pCont; SMqCMCommitOffsetReq commitOffsetReq; SCoder decoder; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index dbf299e8da04d30845985414642d6509d3dbd218..bf378a4d43f02c28479f194f02e457e8c0c5c040 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -50,14 +50,14 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); -static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); -static int32_t mndProcessConnectReq(SMnodeMsg *pReq); -static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq); -static int32_t mndProcessKillConnReq(SMnodeMsg *pReq); -static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); -static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq); +static int32_t mndProcessConnectReq(SNodeMsg *pReq); +static int32_t mndProcessKillQueryReq(SNodeMsg *pReq); +static int32_t mndProcessKillConnReq(SNodeMsg *pReq); +static int32_t mndGetConnsMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetQueryMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { @@ -130,7 +130,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, } static void mndFreeConn(SConnObj *pConn) { - tfree(pConn->pQueries); + taosMemoryFreeClear(pConn->pQueries); mTrace("conn:%d, is destroyed, data:%p", pConn->id, pConn); } @@ -177,8 +177,8 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { } } -static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessConnectReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SUserObj *pUser = NULL; SDbObj *pDb = NULL; SConnObj *pConn = NULL; @@ -206,8 +206,9 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { } if (connReq.db[0]) { - snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); - pDb = mndAcquireDb(pMnode, pReq->db); + char db[TSDB_DB_FNAME_LEN]; + snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); + pDb = mndAcquireDb(pMnode, db); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, connReq.db, terrstr()); @@ -237,8 +238,8 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { if (pRsp == NULL) goto CONN_OVER; tSerializeSConnectRsp(pRsp, contLen, &connectRsp); - pReq->contLen = contLen; - pReq->pCont = pRsp; + pReq->rspLen = contLen; + pReq->pRsp = pRsp; mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, connReq.app); @@ -259,7 +260,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { if (numOfQueries > 0) { if (pConn->pQueries == NULL) { - pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); + pConn->pQueries = taosMemoryCalloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); } pConn->numOfQueries = TMIN(QUERY_SAVE_SIZE, numOfQueries); @@ -275,7 +276,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { #if 0 - SClientHbRsp* pRsp = malloc(sizeof(SClientHbRsp)); + SClientHbRsp* pRsp = taosMemoryMalloc(sizeof(SClientHbRsp)); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -291,7 +292,7 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { SHashObj* pObj = pReq->info; SKv* pKv = taosHashGet(pObj, "mq-tmp", strlen("mq-tmp") + 1); if (pKv == NULL) { - free(pRsp); + taosMemoryFree(pRsp); return NULL; } SMqHbMsg mqHb; @@ -324,7 +325,7 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { taosArrayPush(batchRsp.batchRsps, &innerBatchRsp); } int32_t tlen = taosEncodeSMqHbBatchRsp(NULL, &batchRsp); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { //TODO return NULL; @@ -338,8 +339,8 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { return NULL; } -static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SClientHbBatchReq batchReq = {0}; if (tDeserializeSClientHbBatchReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &batchReq) != 0) { @@ -401,7 +402,7 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { SClientHbRsp *pRsp = mndMqHbBuildRsp(pMnode, pHbReq); if (pRsp != NULL) { taosArrayPush(batchRsp.rsps, pRsp); - free(pRsp); + taosMemoryFree(pRsp); } } } @@ -417,18 +418,18 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0; for (int32_t n = 0; n < kvNum; ++n) { SKv *kv = taosArrayGet(rsp->info, n); - tfree(kv->value); + taosMemoryFreeClear(kv->value); } taosArrayDestroy(rsp->info); } taosArrayDestroy(batchRsp.rsps); - pReq->contLen = tlen; - pReq->pCont = buf; + pReq->rspLen = tlen; + pReq->pRsp = buf; return 0; #if 0 - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SHeartBeatReq *pHeartbeat = pReq->rpcMsg.pCont; @@ -495,13 +496,13 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { mndReleaseConn(pMnode, pConn); pReq->contLen = sizeof(SConnectRsp); - pReq->pCont = pRsp; + pReq->pRsp = pRsp; return 0; #endif } -static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -534,8 +535,8 @@ static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { } } -static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillConnReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -566,8 +567,8 @@ static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { } } -static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetConnsMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -641,8 +642,8 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; @@ -700,8 +701,8 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in return numOfRows; } -static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetQueryMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -815,8 +816,8 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 980e1a5a3fb35ccf4ab14fd4f46dc662db052615..4b19a26bc4f2dc7a6c039b551b17835d347951f0 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -29,14 +29,14 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw); static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew); -static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateQnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateQnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetQnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); -static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq); +static int32_t mndProcessQnodeListReq(SNodeMsg *pReq); int32_t mndInitQnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_QNODE, @@ -128,7 +128,7 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) { QNODE_DECODE_OVER: if (terrno != 0) { mError("qnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -193,7 +193,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -205,10 +205,10 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -220,7 +220,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -232,17 +232,17 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } return 0; } -static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { +static int32_t mndCreateQnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { int32_t code = -1; SQnodeObj qnodeObj = {0}; @@ -268,8 +268,8 @@ CREATE_QNODE_OVER: return code; } -static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateQnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SQnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -343,7 +343,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -355,17 +355,17 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } return 0; } -static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pReq, SQnodeObj *pObj) { +static int32_t mndDropQnode(SMnode *pMnode, SNodeMsg *pReq, SQnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_QNODE, &pReq->rpcMsg); @@ -384,8 +384,8 @@ DROP_QNODE_OVER: return code; } -static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SQnodeObj *pObj = NULL; @@ -432,12 +432,11 @@ DROP_QNODE_OVER: return code; } - -static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq) { +static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { int32_t code = -1; SQnodeListReq qlistReq = {0}; int32_t numOfRows = 0; - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; SQnodeObj *pObj = NULL; SQnodeListRsp qlistRsp = {0}; @@ -475,7 +474,7 @@ static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq) { } int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto QNODE_LIST_OVER; @@ -483,8 +482,8 @@ static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq) { tSerializeSQnodeListRsp(pRsp, rspLen, &qlistRsp); - pReq->contLen = rspLen; - pReq->pCont = pRsp; + pReq->rspLen = rspLen; + pReq->pRsp = pRsp; code = 0; QNODE_LIST_OVER: @@ -494,19 +493,18 @@ QNODE_LIST_OVER: return code; } - -static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateQnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetQnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -545,8 +543,8 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c new file mode 100644 index 0000000000000000000000000000000000000000..e93a0d9b17dc0f5c4b4c69b5df8698c2e4349844 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mndQuery.h" +#include "mndMnode.h" +#include "executor.h" +#include "qworker.h" + +int32_t mndProcessQueryMsg(SNodeMsg *pReq) { + mTrace("message in query queue is processing"); + SMnode *pMnode = pReq->pNode; + SReadHandle handle = {0}; + + switch (pReq->rpcMsg.msgType) { + case TDMT_VND_QUERY: + return qWorkerProcessQueryMsg(&handle, pMnode->pQuery, &pReq->rpcMsg); + case TDMT_VND_QUERY_CONTINUE: + return qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, &pReq->rpcMsg); + default: + mError("unknown msg type:%d in query queue", pReq->rpcMsg.msgType); + return TSDB_CODE_VND_APP_ERROR; + } +} + +int32_t mndProcessFetchMsg(SNodeMsg *pReq) { + mTrace("message in fetch queue is processing"); + SMnode *pMnode = pReq->pNode; + + switch (pReq->rpcMsg.msgType) { + case TDMT_VND_FETCH: + return qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, &pReq->rpcMsg); + case TDMT_VND_DROP_TASK: + return qWorkerProcessDropMsg(pMnode, pMnode->pQuery, &pReq->rpcMsg); + case TDMT_VND_QUERY_HEARTBEAT: + return qWorkerProcessHbMsg(pMnode, pMnode->pQuery, &pReq->rpcMsg); + default: + mError("unknown msg type:%d in fetch queue", pReq->rpcMsg.msgType); + return TSDB_CODE_VND_APP_ERROR; + } +} + +int32_t mndInitQuery(SMnode *pMnode) { + int32_t code = qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb); + if (code) { + return code; + } + + mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_VND_QUERY_CONTINUE, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessFetchMsg); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessFetchMsg); + mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessFetchMsg); + + return 0; +} + +void mndCleanupQuery(SMnode *pMnode) { qWorkerDestroy((void **)&pMnode->pQuery); } + diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index faef757e7625b14d69c395284c5e7a5921711c97..a4dfd293debf69171b65195062f69b591433f87c 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -20,6 +20,7 @@ #include "mndMnode.h" #include "mndOffset.h" #include "mndShow.h" +#include "mndSnode.h" #include "mndStb.h" #include "mndStream.h" #include "mndSubscribe.h" @@ -31,9 +32,95 @@ #include "tname.h" #include "tuuid.h" +extern bool tsStreamSchedV; + +int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) { + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + int32_t size = encoder.pos; + int32_t tlen = sizeof(SMsgHead) + size; + tCoderClear(&encoder); + void* buf = taosMemoryMalloc(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead*)buf)->vgId = htonl(nodeId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, size, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + tCoderClear(&encoder); + + STransAction action = {0}; + memcpy(&action.epSet, pEpSet, sizeof(SEpSet)); + action.pCont = buf; + action.contLen = tlen; + action.msgType = type; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + return -1; + } + return 0; +} + +int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { + int32_t msgLen; + pTask->nodeId = pVgroup->vgId; + pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); + + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epSet = pTask->epSet; + + if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId); + return 0; +} + +SSnodeObj* mndSchedFetchSnode(SMnode* pMnode) { + SSnodeObj* pObj = NULL; + pObj = sdbFetch(pMnode->pSdb, SDB_SNODE, NULL, (void**)&pObj); + return pObj; +} + +int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, + const SSnodeObj* pSnode) { + int32_t msgLen; + + pTask->nodeId = 0; + pTask->epSet = mndAcquireEpFromSnode(pMnode, pSnode); + + plan->execNode.nodeId = 0; + plan->execNode.epSet = pTask->epSet; + + if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_SND_TASK_DEPLOY, 0); + return 0; +} + +SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { + void* pIter = NULL; + SVgObj* pVgroup = NULL; + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != dbUid) { + sdbRelease(pMnode->pSdb, pVgroup); + continue; + } + return pVgroup; + } + return pVgroup; +} + int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; - SVgObj* pVgroup = NULL; SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); if (pPlan == NULL) { terrno = TSDB_CODE_QRY_INVALID_INPUT; @@ -42,162 +129,144 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { ASSERT(pStream->vgNum == 0); int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); - pStream->tasks = taosArrayInit(totLevel, sizeof(SArray)); + ASSERT(totLevel <= 2); + pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); - int32_t msgLen; for (int32_t level = 0; level < totLevel; level++) { - SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask)); + SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level); - int32_t opNum = LIST_LENGTH(inner->pNodeList); - ASSERT(opNum == 1); + ASSERT(LIST_LENGTH(inner->pNodeList) == 1); + + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); - SSubplan* plan = nodesListGetNode(inner->pNodeList, level); - if (level == 0) { - ASSERT(plan->type == SUBPLAN_TYPE_SCAN); + // if (level == totLevel - 1 /* or no snode */) { + if (level == totLevel - 1) { + // last level, source, must assign to vnode + // must be scan type + ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + + // replicate task to each vnode void* pIter = NULL; while (1) { + SVgObj* pVgroup; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); if (pIter == NULL) break; if (pVgroup->dbUid != pStream->dbUid) { sdbRelease(pSdb, pVgroup); continue; } + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + // source part + pTask->sourceType = TASK_SOURCE__SCAN; + + // sink part + if (level == 0) { + // only for inplace + pTask->sinkType = TASK_SINK__SHOW; + pTask->showSink.reserved = 0; + } else { + pTask->sinkType = TASK_SINK__NONE; + } - pStream->vgNum++; - // send to vnode - - SStreamTask* pTask = streamTaskNew(pStream->uid, level); + // dispatch part + if (level == 0) { + pTask->dispatchType = TASK_DISPATCH__NONE; + // if inplace sink, no dispatcher + // if fixed ep, add fixed ep dispatcher + // if shuffle, add shuffle dispatcher + } else { + // add fixed ep dispatcher + int32_t lastLevel = level - 1; + ASSERT(lastLevel == 0); + SArray* pArray = taosArrayGetP(pStream->tasks, lastLevel); + // one merge only + ASSERT(taosArrayGetSize(pArray) == 1); + SStreamTask* lastLevelTask = taosArrayGetP(pArray, lastLevel); + pTask->dispatchMsgType = TDMT_VND_TASK_MERGE_EXEC; + pTask->dispatchType = TASK_DISPATCH__FIXED; + + pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; + pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; + } - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + // exec part + pTask->execType = TASK_EXEC__PIPE; + pTask->exec.parallelizable = 1; + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } - taosArrayPush(taskOneLevel, pTask); - - SCoder encoder; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - int32_t tlen = sizeof(SMsgHead) + encoder.pos; - tCoderClear(&encoder); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMsgHead*)buf)->streamTaskId = pTask->taskId; - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - tCoderClear(&encoder); - - STransAction action = {0}; - action.epSet = plan->execNode.epSet; - action.pCont = buf; - action.contLen = tlen; - action.msgType = TDMT_VND_TASK_DEPLOY; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - rpcFreeCont(buf); return -1; } + sdbRelease(pSdb, pVgroup); + taosArrayPush(taskOneLevel, &pTask); } - } else if (plan->subplanType == SUBPLAN_TYPE_SCAN) { - // duplicatable - - int32_t parallel = 0; - // if no snode, parallel set to fetch thread num in vnode - - // if has snode, set to shared thread num in snode - parallel = SND_SHARED_THREAD_NUM; + } else { + // merge plan - for (int32_t i = 0; i < parallel; i++) { - SStreamTask* pTask = streamTaskNew(pStream->uid, level); + // TODO if has snode, assign to snode - // TODO:get snode id and ep - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + // else, assign to vnode + ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); + SStreamTask* pTask = tNewSStreamTask(pStream->uid); - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } + // source part, currently only support multi source + pTask->sourceType = TASK_SOURCE__PIPE; - taosArrayPush(taskOneLevel, pTask); - - SCoder encoder; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - int32_t tlen = sizeof(SMsgHead) + encoder.pos; - tCoderClear(&encoder); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMsgHead*)buf)->streamTaskId = pTask->taskId; - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - tCoderClear(&encoder); - - STransAction action = {0}; - action.epSet = plan->execNode.epSet; - action.pCont = buf; - action.contLen = tlen; - action.msgType = TDMT_SND_TASK_DEPLOY; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - rpcFreeCont(buf); - return -1; - } - } - } else { - // not duplicatable - SStreamTask* pTask = streamTaskNew(pStream->uid, level); + // sink part + pTask->sinkType = TASK_SINK__SHOW; + /*pTask->sinkType = TASK_SINK__NONE;*/ - // TODO:get snode id and ep - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + // dispatch part + pTask->dispatchType = TASK_DISPATCH__SHUFFLE; + pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC; - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + // exec part + pTask->execType = TASK_EXEC__MERGE; + pTask->exec.parallelizable = 0; + SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->dbUid); + ASSERT(pVgroup); + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } - taosArrayPush(taskOneLevel, pTask); - - SCoder encoder; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - int32_t tlen = sizeof(SMsgHead) + encoder.pos; - tCoderClear(&encoder); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - ((SMsgHead*)buf)->streamTaskId = pTask->taskId; - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); - tEncodeSStreamTask(&encoder, pTask); - tCoderClear(&encoder); - - STransAction action = {0}; - action.epSet = plan->execNode.epSet; - action.pCont = buf; - action.contLen = tlen; - action.msgType = TDMT_SND_TASK_DEPLOY; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - rpcFreeCont(buf); - return -1; + sdbRelease(pSdb, pVgroup); + taosArrayPush(taskOneLevel, &pTask); + } + + taosArrayPush(pStream->tasks, &taskOneLevel); + } + + if (totLevel == 2) { + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pStream->dbUid) { + sdbRelease(pSdb, pVgroup); + continue; } + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + + // source part + pTask->sourceType = TASK_SOURCE__MERGE; + + // sink part + pTask->sinkType = TASK_SINK__SHOW; + + // dispatch part + pTask->dispatchType = TASK_DISPATCH__NONE; + + // exec part + pTask->execType = TASK_EXEC__NONE; + pTask->exec.parallelizable = 0; } - taosArrayPush(pStream->tasks, taskOneLevel); } + + // free memory + qDestroyQueryPlan(pPlan); + return 0; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index ad97888ac5b06d3492c505fdbc696689a72bfc24..dff918f13558b886067a99d8234f069c9b8ffae5 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -22,10 +22,10 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); -static int32_t mndProcessShowReq(SMnodeMsg *pReq); -static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq); +static int32_t mndProcessShowReq(SNodeMsg *pReq); +static int32_t mndProcessRetrieveReq(SNodeMsg *pReq); static bool mndCheckRetrieveFinished(SShowObj *pShow); -static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq); +static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq); int32_t mndInitShow(SMnode *pMnode) { SShowMgmt *pMgmt = &pMnode->showMgmt; @@ -117,8 +117,8 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { taosCacheRelease(pMgmt->cache, (void **)(&pShow), forceRemove); } -static int32_t mndProcessShowReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessShowReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t code = -1; SShowReq showReq = {0}; @@ -146,7 +146,7 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) { } showRsp.showId = pShow->id; - showRsp.tableMeta.pSchemas = calloc(TSDB_MAX_COLUMNS, sizeof(SSchema)); + showRsp.tableMeta.pSchemas = taosMemoryCalloc(TSDB_MAX_COLUMNS, sizeof(SSchema)); if (showRsp.tableMeta.pSchemas == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -161,8 +161,8 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) { int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp); void *pBuf = rpcMallocCont(bufLen); tSerializeSShowRsp(pBuf, bufLen, &showRsp); - pReq->contLen = bufLen; - pReq->pCont = pBuf; + pReq->rspLen = bufLen; + pReq->pRsp = pBuf; mndReleaseShowObj(pShow, false); } else { mndReleaseShowObj(pShow, true); @@ -178,8 +178,8 @@ SHOW_OVER: return code; } -static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; @@ -248,8 +248,8 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision - pReq->pCont = pRsp; - pReq->contLen = size; + pReq->pRsp = pRsp; + pReq->rspLen = size; if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; @@ -263,8 +263,8 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; @@ -289,6 +289,20 @@ static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { mError("failed to process show-meta req since %s", terrstr()); return -1; } + + STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); + pShow->numOfRows = 100; + + int32_t offset = 0; + for(int32_t i = 0; i < meta->numOfColumns; ++i) { + pShow->numOfColumns = meta->numOfColumns; + pShow->offset[i] = offset; + + int32_t bytes = meta->pSchemas[i].bytes; + pShow->rowSize += bytes; + pShow->bytes[i] = bytes; + offset += bytes; + } } else { pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); if (pShow == NULL) { @@ -330,7 +344,7 @@ static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { size = pShow->rowSize * rowsToRead; size += SHOW_STEP_SIZE; - SRetrieveTableRsp *pRsp = rpcMallocCont(size); + SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { mndReleaseShowObj((SShowObj*) pShow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -338,9 +352,18 @@ static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { return -1; } + pRsp->handle = htobe64(pShow->id); + // if free flag is set, client wants to clean the resources if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsRead = (*retrieveFp)(pReq, (SShowObj*) pShow, pRsp->data, rowsToRead); + if (rowsRead < 0) { + terrno = rowsRead; + rpcFreeCont(pRsp); + mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); + mndReleaseShowObj((SShowObj*) pShow, true); + return -1; + } } mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); @@ -348,8 +371,8 @@ static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision - pReq->pCont = pRsp; - pReq->contLen = size; + pReq->pRsp = pRsp; + pReq->rspLen = size; if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; @@ -411,6 +434,8 @@ char *mndShowStr(int32_t showType) { return "show topics"; case TSDB_MGMT_TABLE_FUNC: return "show functions"; + case TSDB_MGMT_TABLE_INDEX: + return "show indexes"; default: return "undefined"; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c new file mode 100644 index 0000000000000000000000000000000000000000..146975aa38851253bc1ca8f656de672a11dc666f --- /dev/null +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -0,0 +1,784 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndSma.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndInfoSchema.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.c" +#include "mndStream.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define TSDB_SMA_VER_NUMBER 1 +#define TSDB_SMA_RESERVE_SIZE 64 + +static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma); +static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw); +static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma); +static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); +static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); +static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq); +static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); +static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); +static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); +static int32_t mndGetSmaMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); + +int32_t mndInitSma(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_SMA, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndSmaActionEncode, + .decodeFp = (SdbDecodeFp)mndSmaActionDecode, + .insertFp = (SdbInsertFp)mndSmaActionInsert, + .updateFp = (SdbUpdateFp)mndSmaActionUpdate, + .deleteFp = (SdbDeleteFp)mndSmaActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessMCreateSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndGetSmaMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupSma(SMnode *pMnode) {} + +static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + int32_t size = sizeof(SSmaObj) + pSma->exprLen + pSma->tagsFilterLen + pSma->sqlLen + pSma->astLen + TSDB_SMA_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_SMA, TSDB_SMA_VER_NUMBER, size); + if (pRaw == NULL) goto _OVER; + + int32_t dataPos = 0; + + SDB_SET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->uid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->stbUid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->dbUid, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->intervalUnit, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->slidingUnit, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->timezone, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->dstVgId, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->interval, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->offset, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->sliding, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->exprLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->tagsFilterLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->sqlLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->astLen, _OVER) + if (pSma->exprLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) + } + if (pSma->tagsFilterLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->tagsFilter, pSma->tagsFilterLen, _OVER) + } + if (pSma->sqlLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->sql, pSma->sqlLen, _OVER) + } + if (pSma->astLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->ast, pSma->astLen, _OVER) + } + + SDB_SET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("sma:%s, failed to encode to raw:%p since %s", pSma->name, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("sma:%s, encode to raw:%p, row:%p", pSma->name, pRaw, pSma); + return pRaw; +} + +static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; + + if (sver != TSDB_SMA_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto _OVER; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SSmaObj)); + if (pRow == NULL) goto _OVER; + + SSmaObj *pSma = sdbGetRowObj(pRow); + if (pSma == NULL) goto _OVER; + + int32_t dataPos = 0; + + SDB_GET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->uid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->stbUid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->dbUid, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->intervalUnit, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->slidingUnit, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->timezone, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->dstVgId, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->interval, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->offset, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->sliding, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->exprLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->tagsFilterLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->sqlLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->astLen, _OVER) + + if (pSma->exprLen > 0) { + pSma->expr = taosMemoryCalloc(pSma->exprLen, 1); + if (pSma->expr == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) + } + + if (pSma->tagsFilterLen > 0) { + pSma->tagsFilter = taosMemoryCalloc(pSma->tagsFilterLen, 1); + if (pSma->tagsFilter == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->tagsFilter, pSma->tagsFilterLen, _OVER) + } + + if (pSma->sqlLen > 0) { + pSma->sql = taosMemoryCalloc(pSma->sqlLen, 1); + if (pSma->sql == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->sql, pSma->sqlLen, _OVER) + } + + if (pSma->astLen > 0) { + pSma->ast = taosMemoryCalloc(pSma->astLen, 1); + if (pSma->ast == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->ast, pSma->astLen, _OVER) + } + + SDB_GET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pRow); + return NULL; + } + + mTrace("sma:%s, decode from raw:%p, row:%p", pSma->name, pRaw, pSma); + return pRow; +} + +static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma) { + mTrace("sma:%s, perform insert action, row:%p", pSma->name, pSma); + return 0; +} + +static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSma) { + mTrace("sma:%s, perform delete action, row:%p", pSma->name, pSma); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pSma->expr); + return 0; +} + +static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew) { + mTrace("sma:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + return 0; +} + +SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName) { + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = sdbAcquire(pSdb, SDB_SMA, smaName); + if (pSma == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + } + return pSma; +} + +void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pSma); +} + +SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *smaName) { + SName name = {0}; + tNameFromString(&name, smaName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + SVCreateTSmaReq req = {0}; + req.tSma.version = 0; + req.tSma.intervalUnit = pSma->intervalUnit; + req.tSma.slidingUnit = pSma->slidingUnit; + req.tSma.timezoneInt = pSma->timezone; + tstrncpy(req.tSma.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); + req.tSma.exprLen = pSma->exprLen; + req.tSma.tagsFilterLen = pSma->tagsFilterLen; + req.tSma.indexUid = pSma->uid; + req.tSma.tableUid = pSma->stbUid; + req.tSma.interval = pSma->interval; + req.tSma.offset = pSma->offset; + req.tSma.sliding = pSma->sliding; + req.tSma.expr = pSma->expr; + req.tSma.tagsFilter = pSma->tagsFilter; + + int32_t contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVCreateTSmaReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + SVDropTSmaReq req = {0}; + req.ver = 0; + req.indexUid = pSma->uid; + tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); + + int32_t contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tDeserializeSVDropTSmaReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +static int32_t mndSetCreateSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateSmaCommitLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pCommitRaw = mndSmaActionEncode(pSma); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + void *pReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_SMA; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCreate, SDbObj *pDb, SStbObj *pStb) { + SSmaObj smaObj = {0}; + memcpy(smaObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); + memcpy(smaObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); + memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN); + smaObj.createdTime = taosGetTimestampMs(); + smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + smaObj.stbUid = pStb->uid; + smaObj.dbUid = pStb->dbUid; + smaObj.intervalUnit = pCreate->intervalUnit; + smaObj.slidingUnit = pCreate->slidingUnit; + smaObj.timezone = pCreate->timezone; + smaObj.dstVgId = pCreate->dstVgId; + smaObj.interval = pCreate->interval; + smaObj.offset = pCreate->offset; + smaObj.sliding = pCreate->sliding; + smaObj.exprLen = pCreate->exprLen; + smaObj.tagsFilterLen = pCreate->tagsFilterLen; + smaObj.sqlLen = pCreate->sqlLen; + smaObj.astLen = pCreate->astLen; + + if (smaObj.exprLen > 0) { + smaObj.expr = taosMemoryMalloc(smaObj.exprLen); + if (smaObj.expr == NULL) goto _OVER; + memcpy(smaObj.expr, pCreate->expr, smaObj.exprLen); + } + + if (smaObj.tagsFilterLen > 0) { + smaObj.tagsFilter = taosMemoryMalloc(smaObj.tagsFilterLen); + if (smaObj.tagsFilter == NULL) goto _OVER; + memcpy(smaObj.tagsFilter, pCreate->tagsFilter, smaObj.tagsFilterLen); + } + + if (smaObj.sqlLen > 0) { + smaObj.sql = taosMemoryMalloc(smaObj.sqlLen); + if (smaObj.sql == NULL) goto _OVER; + memcpy(smaObj.sql, pCreate->sql, smaObj.sqlLen); + } + + if (smaObj.astLen > 0) { + smaObj.ast = taosMemoryMalloc(smaObj.astLen); + if (smaObj.ast == NULL) goto _OVER; + memcpy(smaObj.ast, pCreate->ast, smaObj.astLen); + } + + SStreamObj streamObj = {0}; + tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); + tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN); + streamObj.createTime = taosGetTimestampMs(); + streamObj.updateTime = streamObj.createTime; + streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); + streamObj.dbUid = pDb->uid; + streamObj.version = 1; + streamObj.sql = pCreate->sql; + /*streamObj.physicalPlan = "";*/ + streamObj.logicalPlan = "not implemented"; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, &pReq->rpcMsg); + if (pTrans == NULL) goto _OVER; + + mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); + mndTransSetDbInfo(pTrans, pDb); + + if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; + if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; + if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + if (pCreate->name[0] == 0) return -1; + if (pCreate->stb[0] == 0) return -1; + if (pCreate->igExists < 0 || pCreate->igExists > 1) return -1; + if (pCreate->intervalUnit < 0) return -1; + if (pCreate->slidingUnit < 0) return -1; + if (pCreate->timezone < 0) return -1; + if (pCreate->dstVgId < 0) return -1; + if (pCreate->interval < 0) return -1; + if (pCreate->offset < 0) return -1; + if (pCreate->sliding < 0) return -1; + if (pCreate->exprLen < 0) return -1; + if (pCreate->tagsFilterLen < 0) return -1; + if (pCreate->sqlLen < 0) return -1; + if (pCreate->astLen < 0) return -1; + if (pCreate->exprLen != 0 && strlen(pCreate->expr) + 1 != pCreate->exprLen) return -1; + if (pCreate->tagsFilterLen != 0 && strlen(pCreate->tagsFilter) + 1 != pCreate->tagsFilterLen) return -1; + if (pCreate->sqlLen != 0 && strlen(pCreate->sql) + 1 != pCreate->sqlLen) return -1; + if (pCreate->astLen != 0 && strlen(pCreate->ast) + 1 != pCreate->astLen) return -1; + + SName smaName = {0}; + if (tNameFromString(&smaName, pCreate->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE) < 0) return -1; + if (*(char *)tNameGetTableName(&smaName) == 0) return -1; + + terrno = 0; + return 0; +} + +static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SStbObj *pStb = NULL; + SSmaObj *pSma = NULL; + SStreamObj *pStream = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SMCreateSmaReq createReq = {0}; + + if (tDeserializeSMCreateSmaReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mDebug("sma:%s, start to create", createReq.name); + if (mndCheckCreateSmaReq(&createReq) != 0) { + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, createReq.stb); + if (pStb == NULL) { + mError("sma:%s, failed to create since stb:%s not exist", createReq.name, createReq.stb); + goto _OVER; + } + + pStream = mndAcquireStream(pMnode, createReq.name); + if (pStream != NULL) { + mError("sma:%s, failed to create since stream:%s already exist", createReq.name, createReq.name); + goto _OVER; + } + + pSma = mndAcquireSma(pMnode, createReq.name); + if (pSma != NULL) { + if (createReq.igExists) { + mDebug("sma:%s, already exist in sma:%s, ignore exist is set", createReq.name, pSma->name); + code = 0; + goto _OVER; + } else { + terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + goto _OVER; + } + } + + pDb = mndAcquireDbBySma(pMnode, createReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + code = mndCreateSma(pMnode, pReq, &createReq, pDb, pStb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("sma:%s, failed to create since %s", createReq.name, terrstr()); + } + + mndReleaseStb(pMnode, pStb); + mndReleaseSma(pMnode, pSma); + mndReleaseStream(pMnode, pStream); + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + tFreeSMCreateSmaReq(&createReq); + + return code; +} + +static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + +static int32_t mndSetDropSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropSmaCommitLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pCommitRaw = mndSmaActionEncode(pSma); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + int32_t contLen = 0; + void *pReq = mndBuildVDropSmaReq(pMnode, pVgroup, pSma, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_DROP_SMA; + action.acceptableCode = TSDB_CODE_VND_SMA_NOT_EXIST; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndDropSma(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SSmaObj *pSma) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_SMA, &pReq->rpcMsg); + if (pTrans == NULL) goto _OVER; + + mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); + mndTransSetDbInfo(pTrans, pDb); + + if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserObj *pUser = NULL; + SDbObj *pDb = NULL; + SSmaObj *pSma = NULL; + SMDropSmaReq dropReq = {0}; + + if (tDeserializeSMDropSmaReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mDebug("sma:%s, start to drop", dropReq.name); + + pSma = mndAcquireSma(pMnode, dropReq.name); + if (pSma == NULL) { + if (dropReq.igNotExists) { + mDebug("sma:%s, not exist, ignore not exist is set", dropReq.name); + code = 0; + goto _OVER; + } else { + terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + goto _OVER; + } + } + + pDb = mndAcquireDbBySma(pMnode, dropReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + code = mndDropSma(pMnode, pReq, pDb, pSma); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("sma:%s, failed to drop since %s", dropReq.name, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseSma(pMnode, pSma); + mndReleaseUser(pMnode, pUser); + + return code; +} + +static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + +static int32_t mndGetSmaMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchemas; + + pShow->bytes[cols] = TSDB_INDEX_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "stb"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pMeta->numOfColumns = cols; + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_SMA); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbName, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SSmaObj *pSma = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[TSDB_DB_FNAME_LEN] = {0}; + + SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma); + if (pShow->pIter == NULL) break; + + if (pSma->dbUid != pDb->uid) { + sdbRelease(pSdb, pSma); + continue; + } + + cols = 0; + + SName smaName = {0}; + tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, (char *)tNameGetTableName(&smaName)); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pSma->createdTime; + cols++; + + SName stbName = {0}; + tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, (char *)tNameGetTableName(&stbName)); + cols++; + + numOfRows++; + sdbRelease(pSdb, pSma); + } + + mndReleaseDb(pMnode, pDb); + pShow->numOfReads += numOfRows; + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 6040aa088c8debc93a6149866e28da363762838c..5e0d9fae9a4bccbd382355ad2897b99c13ea8929 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -29,12 +29,12 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw); static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew); -static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateSnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateSnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetSnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); int32_t mndInitSnode(SMnode *pMnode) { @@ -60,6 +60,15 @@ int32_t mndInitSnode(SMnode *pMnode) { void mndCleanupSnode(SMnode *pMnode) {} +SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode) { + SEpSet epSet; + memcpy(epSet.eps->fqdn, pSnode->pDnode->fqdn, 128); + epSet.eps->port = pSnode->pDnode->port; + epSet.numOfEps = 1; + epSet.inUse = 0; + return epSet; +} + static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { SSnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_SNODE, &snodeId); if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -126,7 +135,7 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) { SNODE_DECODE_OVER: if (terrno != 0) { mError("snode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -191,7 +200,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -203,10 +212,10 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -218,7 +227,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -230,17 +239,17 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } return 0; } -static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { +static int32_t mndCreateSnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { int32_t code = -1; SSnodeObj snodeObj = {0}; @@ -267,8 +276,8 @@ CREATE_SNODE_OVER: return code; } -static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateSnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SSnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -343,7 +352,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -355,17 +364,17 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } return 0; } -static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pReq, SSnodeObj *pObj) { +static int32_t mndDropSnode(SMnode *pMnode, SNodeMsg *pReq, SSnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_SNODE, &pReq->rpcMsg); @@ -385,8 +394,8 @@ DROP_SNODE_OVER: return code; } -static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SSnodeObj *pObj = NULL; @@ -433,18 +442,18 @@ DROP_SNODE_OVER: return code; } -static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateSnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetSnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -483,8 +492,8 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d8091dfa760bc939d60867852d74aae459aa207a..fdd03be710a5d261526c0db32d8a42b65afcfc5b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -13,35 +13,34 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "mndStb.h" #include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndInfoSchema.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "mndInfoSchema.h" #include "tname.h" -#define TSDB_STB_VER_NUMBER 1 +#define TSDB_STB_VER_NUMBER 1 #define TSDB_STB_RESERVE_SIZE 64 static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); -static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq); -static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq); -static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq); -static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq); -static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq); +static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq); +static int32_t mndProcessMDropStbReq(SNodeMsg *pReq); +static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessVAlterStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessVDropStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessTableMetaReq(SNodeMsg *pReq); +static int32_t mndGetStbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); int32_t mndInitStb(SMnode *pMnode) { @@ -73,7 +72,8 @@ void mndCleanupStb(SMnode *pMnode) {} SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE; + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) + + TSDB_STB_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); if (pRaw == NULL) goto STB_ENCODE_OVER; @@ -86,8 +86,14 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->version, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->delay, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->ttl, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, STB_ENCODE_OVER) for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; @@ -105,7 +111,17 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } - SDB_SET_BINARY(pRaw, dataPos, pStb->comment, TSDB_STB_COMMENT_LEN, STB_ENCODE_OVER) + for (int32_t i = 0; i < pStb->numOfSmas; ++i) { + SSchema *pSchema = &pStb->pSmas[i]; + SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) + } + + if (pStb->commentLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_ENCODE_OVER) + } SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, STB_ENCODE_OVER) @@ -148,12 +164,21 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->version, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, STB_DECODE_OVER) + int32_t xFilesFactor = 0; + SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, STB_DECODE_OVER) + pStb->xFilesFactor = xFilesFactor / 10000.0f; + SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->delay, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, STB_DECODE_OVER) - pStb->pColumns = calloc(pStb->numOfColumns, sizeof(SSchema)); - pStb->pTags = calloc(pStb->numOfTags, sizeof(SSchema)); - if (pStb->pColumns == NULL || pStb->pTags == NULL) { + pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); + pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); + pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema)); + if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) { goto STB_DECODE_OVER; } @@ -173,7 +198,19 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } - SDB_GET_BINARY(pRaw, dataPos, pStb->comment, TSDB_STB_COMMENT_LEN, STB_DECODE_OVER) + for (int32_t i = 0; i < pStb->numOfSmas; ++i) { + SSchema *pSchema = &pStb->pSmas[i]; + SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) + } + + if (pStb->commentLen > 0) { + pStb->comment = taosMemoryCalloc(pStb->commentLen, 1); + if (pStb->comment == NULL) goto STB_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_DECODE_OVER) + } SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_DECODE_OVER) terrno = 0; @@ -181,9 +218,10 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { STB_DECODE_OVER: if (terrno != 0) { mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr()); - tfree(pStb->pColumns); - tfree(pStb->pTags); - tfree(pRow); + taosMemoryFreeClear(pStb->pColumns); + taosMemoryFreeClear(pStb->pTags); + taosMemoryFreeClear(pStb->comment); + taosMemoryFreeClear(pRow); return NULL; } @@ -198,8 +236,9 @@ static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb); - tfree(pStb->pColumns); - tfree(pStb->pTags); + taosMemoryFreeClear(pStb->pColumns); + taosMemoryFreeClear(pStb->pTags); + taosMemoryFreeClear(pStb->comment); return 0; } @@ -209,9 +248,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { taosWLockLatch(&pOld->lock); if (pOld->numOfColumns < pNew->numOfColumns) { - void *pColumns = malloc(pNew->numOfColumns * sizeof(SSchema)); + void *pColumns = taosMemoryMalloc(pNew->numOfColumns * sizeof(SSchema)); if (pColumns != NULL) { - free(pOld->pColumns); + taosMemoryFree(pOld->pColumns); pOld->pColumns = pColumns; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -221,9 +260,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } if (pOld->numOfTags < pNew->numOfTags) { - void *pTags = malloc(pNew->numOfTags * sizeof(SSchema)); + void *pTags = taosMemoryMalloc(pNew->numOfTags * sizeof(SSchema)); if (pTags != NULL) { - free(pOld->pTags); + taosMemoryFree(pOld->pTags); pOld->pTags = pTags; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -232,6 +271,30 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } } + if (pOld->numOfSmas < pNew->numOfSmas) { + void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema)); + if (pSmas != NULL) { + taosMemoryFree(pOld->pSmas); + pOld->pSmas = pSmas; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); + } + } + + if (pOld->commentLen < pNew->commentLen) { + void *comment = taosMemoryMalloc(pNew->commentLen); + if (comment != NULL) { + taosMemoryFree(pOld->comment); + pOld->comment = comment; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); + } + } + pOld->updateTime = pNew->updateTime; pOld->version = pNew->version; pOld->nextColId = pNew->nextColId; @@ -239,7 +302,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->numOfTags = pNew->numOfTags; memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema)); memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema)); - memcpy(pOld->comment, pNew->comment, TSDB_STB_COMMENT_LEN); + if (pNew->commentLen != 0) { + memcpy(pOld->comment, pNew->comment, TSDB_STB_COMMENT_LEN); + } taosWUnLockLatch(&pOld->lock); return 0; } @@ -271,9 +336,12 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { SName name = {0}; tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFName); SVCreateTbReq req = {0}; req.ver = 0; + req.dbFName = dbFName; req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; @@ -285,7 +353,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.stbCfg.pTagSchema = pStb->pTags; int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = malloc(contLen); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -312,7 +380,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, req.suid = pStb->uid; int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = malloc(contLen); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -440,7 +508,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_CREATE_STB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -479,7 +547,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -490,7 +558,17 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { +static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { + for (int32_t col = 0; col < pStb->numOfColumns; col++) { + SSchema *pSchema = &pStb->pColumns[col]; + if (strcasecmp(pStb->pColumns[col].name, colName) == 0) { + return pSchema; + } + } + return NULL; +} + +static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); @@ -500,12 +578,24 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCr stbObj.dbUid = pDb->uid; stbObj.version = 1; stbObj.nextColId = 1; + stbObj.ttl = pCreate->ttl; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; + stbObj.numOfSmas = pCreate->numOfSmas; + stbObj.commentLen = pCreate->commentLen; + if (stbObj.commentLen > 0) { + stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); + if (stbObj.comment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen); + } - stbObj.pColumns = malloc(stbObj.numOfColumns * sizeof(SSchema)); - stbObj.pTags = malloc(stbObj.numOfTags * sizeof(SSchema)); - if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { + stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema)); + stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema)); + stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema)); + if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -530,6 +620,18 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCr stbObj.nextColId++; } + for (int32_t i = 0; i < stbObj.numOfSmas; ++i) { + SField *pField = taosArrayGet(pCreate->pSmas, i); + SSchema *pSchema = &stbObj.pSmas[i]; + SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name); + if (pColSchema == NULL) { + mError("stb:%s, sma:%s not found in columns", stbObj.name, pSchema->name); + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + memcpy(pSchema, pColSchema, sizeof(SSchema)); + } + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_STB_OVER; @@ -551,8 +653,8 @@ CREATE_STB_OVER: return code; } -static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SStbObj *pTopicStb = NULL; SStbObj *pStb = NULL; @@ -623,7 +725,7 @@ CREATE_STB_OVER: return code; } -static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -675,8 +777,8 @@ static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *col } static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { - pNew->pTags = calloc(pNew->numOfTags, sizeof(SSchema)); - pNew->pColumns = calloc(pNew->numOfColumns, sizeof(SSchema)); + pNew->pTags = taosMemoryCalloc(pNew->numOfTags, sizeof(SSchema)); + pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema)); if (pNew->pTags == NULL || pNew->pColumns == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -969,7 +1071,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_ALTER_STB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -980,7 +1082,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndAlterStb(SMnode *pMnode, SMnodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { +static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); @@ -1038,13 +1140,13 @@ static int32_t mndAlterStb(SMnode *pMnode, SMnodeMsg *pReq, const SMAltertbReq * ALTER_STB_OVER: mndTransDrop(pTrans); - tfree(stbObj.pTags); - tfree(stbObj.pColumns); + taosMemoryFreeClear(stbObj.pTags); + taosMemoryFreeClear(stbObj.pColumns); return code; } -static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SStbObj *pStb = NULL; @@ -1096,7 +1198,7 @@ ALTER_STB_OVER: return code; } -static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVAlterStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -1149,7 +1251,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * action.msgType = TDMT_VND_DROP_STB; action.acceptableCode = TSDB_CODE_VND_TB_NOT_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -1160,9 +1262,9 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * return 0; } -static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { +static int32_t mndDropStb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_STB, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_STB, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_STB_OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); @@ -1180,8 +1282,8 @@ DROP_STB_OVER: return code; } -static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMDropStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDbObj *pDb = NULL; @@ -1237,7 +1339,7 @@ DROP_STB_OVER: return code; } -static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVDropStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -1246,7 +1348,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa taosRLockLatch(&pStb->lock); int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - pRsp->pSchemas = calloc(totalCols, sizeof(SSchema)); + pRsp->pSchemas = taosMemoryCalloc(totalCols, sizeof(SSchema)); if (pRsp->pSchemas == NULL) { taosRUnLockLatch(&pStb->lock); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1311,8 +1413,8 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } -static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessTableMetaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; STableInfoReq infoReq = {0}; STableMetaRsp metaRsp = {0}; @@ -1347,8 +1449,8 @@ static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); - pReq->pCont = pRsp; - pReq->contLen = rspLen; + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; code = 0; mDebug("stb:%s.%s, meta is retrieved", infoReq.dbFName, infoReq.tbName); @@ -1396,7 +1498,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int return -1; } - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { tFreeSTableMetaBatchRsp(&batchMetaRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1436,8 +1538,8 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs return 0; } -static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetStbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfStbs(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -1490,7 +1592,7 @@ static void mndExtractTableName(char *tableId, char *name) { int32_t pos = -1; int32_t num = 0; for (pos = 0; tableId[pos] != 0; ++pos) { - if (tableId[pos] == '.') num++; + if (tableId[pos] == TS_PATH_DELIMITER[0]) num++; if (num == 2) break; } @@ -1499,8 +1601,8 @@ static void mndExtractTableName(char *tableId, char *name) { } } -static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStbObj *pStb = NULL; @@ -1508,34 +1610,37 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 char *pWrite; char prefix[TSDB_DB_FNAME_LEN] = {0}; - SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) return 0; - - tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = (int32_t)strlen(prefix); + SDbObj* pDb = NULL; + if (strlen(pShow->db) > 0) { + pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return terrno; + } while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); if (pShow->pIter == NULL) break; - if (pStb->dbUid != pDb->uid) { - if (strncmp(pStb->db, pDb->name, prefixLen) == 0) { - mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid); - } - + if (pDb != NULL && pStb->dbUid != pDb->uid) { sdbRelease(pSdb, pStb); continue; } cols = 0; + SName name = {0}; char stbName[TSDB_TABLE_NAME_LEN] = {0}; - tstrncpy(stbName, pStb->name + prefixLen, TSDB_TABLE_NAME_LEN); + mndExtractTableName(pStb->name, stbName); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_TO_VARSTR(pWrite, stbName); cols++; + char db[TSDB_DB_NAME_LEN] = {0}; + tNameFromString(&name, pStb->db, T_NAME_ACCT|T_NAME_DB); + tNameGetDbName(&name, db); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, db); + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int64_t *)pWrite = pStb->createdTime; cols++; @@ -1548,11 +1653,30 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 *(int32_t *)pWrite = pStb->numOfTags; cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = 0; // number of tables + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pStb->updateTime; // number of tables + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pStb->commentLen != 0) { + STR_TO_VARSTR(pWrite, pStb->comment); + } else { + STR_TO_VARSTR(pWrite, ""); + } + cols++; + numOfRows++; sdbRelease(pSdb, pStb); } - mndReleaseDb(pMnode, pDb); + if (pDb != NULL) { + mndReleaseDb(pMnode, pDb); + } + pShow->numOfReads += numOfRows; mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index af0f354c11dc668353582e9a9cf862165df25e10..dd62bc0364eb35b8cd4697c919afde86d9e5f1ad 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -14,6 +14,7 @@ */ #include "mndStream.h" +#include "parser.h" #include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" @@ -21,6 +22,7 @@ #include "mndScheduler.h" #include "mndShow.h" #include "mndStb.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -32,12 +34,13 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); -static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq); -/*static int32_t mndProcessDropStreamReq(SMnodeMsg *pReq);*/ -/*static int32_t mndProcessDropStreamInRsp(SMnodeMsg *pRsp);*/ -static int32_t mndProcessStreamMetaReq(SMnodeMsg *pReq); -static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq); +static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp); +/*static int32_t mndProcessDropStreamReq(SNodeMsg *pReq);*/ +/*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/ +static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq); +static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitStream(SMnode *pMnode) { @@ -50,6 +53,8 @@ int32_t mndInitStream(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndStreamActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); + mndSetMsgHandle(pMnode, TDMT_VND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); + mndSetMsgHandle(pMnode, TDMT_SND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/ /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ @@ -68,7 +73,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - if (tEncodeSStreamObj(NULL, pStream) < 0) { + if (tEncodeSStreamObj(&encoder, pStream) < 0) { tCoderClear(&encoder); goto STREAM_ENCODE_OVER; } @@ -79,11 +84,11 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SSdbRaw *pRaw = sdbAllocRaw(SDB_STREAM, MND_STREAM_VER_NUMBER, size); if (pRaw == NULL) goto STREAM_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto STREAM_ENCODE_OVER; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); - if (tEncodeSStreamObj(NULL, pStream) < 0) { + if (tEncodeSStreamObj(&encoder, pStream) < 0) { tCoderClear(&encoder); goto STREAM_ENCODE_OVER; } @@ -97,7 +102,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { terrno = TSDB_CODE_SUCCESS; STREAM_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -130,12 +135,12 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { int32_t tlen; int32_t dataPos = 0; SDB_GET_INT32(pRaw, dataPos, &tlen, STREAM_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto STREAM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER); SCoder decoder; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, NULL, 0, TD_DECODER); + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, tlen + 1, TD_DECODER); if (tDecodeSStreamObj(&decoder, pStream) < 0) { goto STREAM_DECODE_OVER; } @@ -143,10 +148,10 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; STREAM_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -166,7 +171,7 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) { static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) { mTrace("stream:%s, perform update action", pOldStream->name); - atomic_exchange_32(&pOldStream->updateTime, pNewStream->updateTime); + atomic_exchange_64(&pOldStream->updateTime, pNewStream->updateTime); atomic_exchange_32(&pOldStream->version, pNewStream->version); taosWLockLatch(&pOldStream->lock); @@ -191,6 +196,11 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { sdbRelease(pSdb, pStream); } +static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) { SName name = {0}; tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -209,40 +219,103 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { return 0; } -static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { +static int32_t mndStreamGetPlanString(const char *ast, char **pStr) { + if (NULL == ast) { + return TSDB_CODE_SUCCESS; + } + + SNode *pAst = NULL; + int32_t code = nodesStringToNode(ast, &pAst); + + SQueryPlan *pPlan = NULL; + if (TSDB_CODE_SUCCESS == code) { + SPlanContext cxt = { + .pAstRoot = pAst, + .topicQuery = false, + .streamQuery = true, + }; + code = qCreateQueryPlan(&cxt, &pPlan, NULL); + } + + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pPlan, false, pStr, NULL); + } + nodesDestroyNode(pAst); + nodesDestroyNode(pPlan); + terrno = code; + return code; +} + +int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { + SNode *pAst = NULL; +#if 1 // TODO: remove debug info later + printf("ast = %s\n", ast); +#endif + if (nodesStringToNode(ast, &pAst) < 0) { + return -1; + } +#if 1 + SSchemaWrapper sw = {0}; + qExtractResultSchema(pAst, (int32_t*)&sw.nCols, &sw.pSchema); + + printf("|"); + for (int i = 0; i < sw.nCols; i++) { + printf(" %15s |", (char *)sw.pSchema[i].name); + } + printf("\n=======================================================\n"); + + pStream->ColAlias = NULL; +#endif + + if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, &pStream->physicalPlan)) { + mError("topic:%s, failed to get plan since %s", pStream->name, terrstr()); + return -1; + } + + if (mndScheduleStream(pMnode, pTrans, pStream) < 0) { + mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr()); + return -1; + } + mDebug("trans:%d, used to create stream:%s", pTrans->id, pStream->name); + + SSdbRaw *pRedoRaw = mndStreamActionEncode(pStream); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + return 0; +} + +static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { mDebug("stream:%s to create", pCreate->name); SStreamObj streamObj = {0}; tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN); + tstrncpy(streamObj.outputSTbName, pCreate->outputSTbName, TSDB_TABLE_FNAME_LEN); streamObj.createTime = taosGetTimestampMs(); streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); streamObj.dbUid = pDb->uid; streamObj.version = 1; streamObj.sql = pCreate->sql; - streamObj.physicalPlan = pCreate->physicalPlan; - streamObj.logicalPlan = pCreate->logicalPlan; + /*streamObj.physicalPlan = "";*/ + streamObj.logicalPlan = "not implemented"; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); - if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { - mError("stream:%ld, schedule stream since %s", streamObj.uid, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) { + mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -254,8 +327,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamR return 0; } -static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SStreamObj *pStream = NULL; SDbObj *pDb = NULL; @@ -346,8 +419,8 @@ static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfS return 0; } -static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfStreams(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -390,8 +463,8 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 1dcd07829d32839804ab425d41a1cae7db7c86c3..08a88a19ec2f252ee91b6a260973fb8883e7544a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -48,14 +48,14 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub); -static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg); -static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); -static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); -static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg); -static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg); +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); +static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg); +static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg); +static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg); static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, const SMqConsumerEp *pConsumerEp); @@ -94,7 +94,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { tDeleteSMqSubscribeObj(pSub); - free(pSub); + taosMemoryFree(pSub); return NULL; } @@ -102,7 +102,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSMqSubscribeObj(pSub); - free(pSub); + taosMemoryFree(pSub); return NULL; } #endif @@ -118,7 +118,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume }; int32_t tlen = tEncodeSMqMVRebReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -157,7 +157,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } @@ -169,7 +169,7 @@ static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsum req.consumerId = pConsumerEp->consumerId; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -203,7 +203,7 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } @@ -211,8 +211,8 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq } #if 0 -static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; uint8_t *str = pMsg->rpcMsg.pCont; SMqCMResetOffsetReq req; @@ -229,7 +229,7 @@ static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg) { SMqOffset *pOffset = &req.offsets[i]; SMqVgOffsets *pVgOffset = taosHashGet(pHash, &pOffset->vgId, sizeof(int32_t)); if (pVgOffset == NULL) { - pVgOffset = malloc(sizeof(SMqVgOffsets)); + pVgOffset = taosMemoryMalloc(sizeof(SMqVgOffsets)); if (pVgOffset == NULL) { return -1; } @@ -249,14 +249,14 @@ static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg) { } #endif -static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; SMqCMGetSubEpRsp rsp = {0}; int64_t consumerId = be64toh(pReq->consumerId); int32_t epoch = ntohl(pReq->epoch); - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); if (pConsumer == NULL) { terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; @@ -272,7 +272,6 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/ strcpy(rsp.cgroup, pReq->cgroup); - rsp.consumerId = consumerId; if (epoch != pConsumer->epoch) { mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch); SArray *pTopics = pConsumer->currentTopics; @@ -322,13 +321,14 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { } ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; ((SMqRspHead *)buf)->epoch = pConsumer->epoch; + ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); tDeleteSMqCMGetSubEpRsp(&rsp); mndReleaseConsumer(pMnode, pConsumer); - pMsg->pCont = buf; - pMsg->contLen = tlen; + pMsg->pRsp = buf; + pMsg->rspLen = tlen; return 0; } @@ -337,27 +337,27 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) while (key[i] != TMQ_SEPARATOR) { i++; } - memcpy(topic, key, i - 1); - topic[i] = 0; - strcpy(cgroup, &key[i + 1]); + memcpy(cgroup, key, i); + cgroup[i] = 0; + strcpy(topic, &key[i + 1]); return 0; } static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key)); + SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); if (pRebSub == NULL) { pRebSub = tNewSMqRebSubscribe(key); if (pRebSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - taosHashPut(pHash, key, strlen(key), pRebSub, sizeof(SMqRebSubscribe)); + taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe)); } return pRebSub; } -static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer; void *pIter = NULL; @@ -407,7 +407,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics); for (int32_t i = 0; i < removeSz; i++) { char *topicName = taosArrayGet(pConsumer->recentRemovedTopics, i); - free(topicName); + taosMemoryFree(topicName); } taosArrayClear(pConsumer->recentRemovedTopics); } @@ -420,7 +420,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg), }; - pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); @@ -428,8 +428,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg); void *pIter = NULL; @@ -441,6 +441,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { if (pIter == NULL) break; SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); + taosMemoryFreeClear(pRebSub->key); mInfo("mq rebalance subscription: %s", pSub->key); @@ -503,7 +504,8 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); } - mInfo("mq consumer:%" PRId64 ", status change from %d to %d", pRebConsumer->consumerId, status, pRebConsumer->status); + mInfo("mq consumer:%" PRId64 ", status change from %d to %d", pRebConsumer->consumerId, status, + pRebConsumer->status); SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); @@ -537,14 +539,14 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { mndSplitSubscribeKey(pSub->key, topic, cgroup); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 "", pConsumerEp->vgId, topic, - pConsumerEp->consumerId); + mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 " cgroup: %s", pConsumerEp->vgId, + topic, pConsumerEp->consumerId, cgroup); mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); mndReleaseTopic(pMnode, pTopic); } else { - mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "", pConsumerEp->vgId, - pConsumerEp->oldConsumerId, pConsumerEp->consumerId); + mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "", + pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId); mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); } @@ -760,8 +762,8 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { } mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); - tfree(topic); - tfree(cgroup); + taosMemoryFreeClear(topic); + taosMemoryFreeClear(cgroup); } // rebalance condition2 : imbalance assignment } @@ -834,7 +836,7 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT strcpy(req.cgroup, cgroup); strcpy(req.topicName, pTopic->name); int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -858,7 +860,7 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } return 0; @@ -875,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto SUB_ENCODE_OVER; void *abuf = buf; @@ -890,7 +892,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_SUCCESS; SUB_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -923,7 +925,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -935,10 +937,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; SUB_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -994,8 +996,8 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } -static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; char *msgStr = pMsg->rpcMsg.pCont; SCMSubscribeReq subscribe; tDeserializeSCMSubscribeReq(msgStr, &subscribe); @@ -1099,7 +1101,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, newTopicName); bool createSub = false; if (pSub == NULL) { - mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup, newTopicName); + mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup, + newTopicName); pSub = mndCreateSubscription(pMnode, pTopic, cgroup); createSub = true; @@ -1137,7 +1140,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } } - if (oldSub) taosArrayDestroyEx(oldSub, free); + if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void*))taosMemoryFree); // persist consumerObj SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); @@ -1156,7 +1159,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 453535c6e762211fcd5d5ff9687197a2af9cd983..d6c1b6c94fae8bb5a1f098c1eb956579b0b6e7cb 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -79,8 +79,8 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { return pCont; } -static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { - SMnode* pMnode = pReq->pMnode; +static int32_t mndProcessTelemTimer(SNodeMsg* pReq) { + SMnode* pMnode = pReq->pNode; STelemMgmt* pMgmt = &pMnode->telemMgmt; if (!pMgmt->enable) return 0; @@ -88,7 +88,7 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { char* pCont = mndBuildTelemetryReport(pMnode); if (pCont != NULL) { taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont), HTTP_FLAT); - free(pCont); + taosMemoryFree(pCont); } taosWUnLockLatch(&pMgmt->lock); return 0; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index e913cae1ace56445bb44c46a77f711b21000507d..625c9eb733a98699df41f5766be07087cc1f40c1 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -31,12 +31,12 @@ static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic); -static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq); -static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq); -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp); -static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq); -static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq); +static int32_t mndProcessDropTopicReq(SNodeMsg *pReq); +static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp); +static int32_t mndProcessTopicMetaReq(SNodeMsg *pReq); +static int32_t mndGetTopicMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); int32_t mndInitTopic(SMnode *pMnode) { @@ -129,11 +129,11 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); - pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char)); + pTopic->sql = taosMemoryCalloc(pTopic->sqlLen + 1, sizeof(char)); SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - pTopic->logicalPlan = calloc(len + 1, sizeof(char)); + pTopic->logicalPlan = taosMemoryCalloc(len + 1, sizeof(char)); if (pTopic->logicalPlan == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; @@ -141,9 +141,9 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - pTopic->physicalPlan = calloc(len + 1, sizeof(char)); + pTopic->physicalPlan = taosMemoryCalloc(len + 1, sizeof(char)); if (pTopic->physicalPlan == NULL) { - free(pTopic->logicalPlan); + taosMemoryFree(pTopic->logicalPlan); terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; } @@ -156,7 +156,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { TOPIC_DECODE_OVER: if (terrno != TSDB_CODE_SUCCESS) { mError("topic:%s, failed to decode from raw:%p since %s", pTopic->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -176,7 +176,7 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) { static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) { mTrace("topic:%s, perform update action", pOldTopic->name); - atomic_exchange_32(&pOldTopic->updateTime, pNewTopic->updateTime); + atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); atomic_exchange_32(&pOldTopic->version, pNewTopic->version); taosWLockLatch(&pOldTopic->lock); @@ -214,7 +214,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { int32_t contLen = sizeof(SDDropTopicReq); - SDDropTopicReq *pDrop = calloc(1, contLen); + SDDropTopicReq *pDrop = taosMemoryCalloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -236,17 +236,17 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { return 0; } -static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) { +static int32_t mndGetPlanString(const SCMCreateTopicReq *pCreate, char **pStr) { if (NULL == pCreate->ast) { return TSDB_CODE_SUCCESS; } - SNode* pAst = NULL; + SNode *pAst = NULL; int32_t code = nodesStringToNode(pCreate->ast, &pAst); - SQueryPlan* pPlan = NULL; + SQueryPlan *pPlan = NULL; if (TSDB_CODE_SUCCESS == code) { - SPlanContext cxt = { .pAstRoot = pAst, .streamQuery = true }; + SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; code = qCreateQueryPlan(&cxt, &pPlan, NULL); } @@ -259,7 +259,7 @@ static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) { return code; } -static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); @@ -274,7 +274,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq topicObj.logicalPlan = ""; topicObj.sqlLen = strlen(pCreate->sql); - char* pPlanStr = NULL; + char *pPlanStr = NULL; if (TSDB_CODE_SUCCESS != mndGetPlanString(pCreate, &pPlanStr)) { mError("topic:%s, failed to get plan since %s", pCreate->name, terrstr()); return -1; @@ -286,7 +286,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); return -1; } mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); @@ -294,7 +294,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return -1; } @@ -302,18 +302,18 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return -1; } - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return 0; } -static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SMqTopicObj *pTopic = NULL; SDbObj *pDb = NULL; @@ -377,7 +377,7 @@ CREATE_TOPIC_OVER: return code; } -static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic) { +static int32_t mndDropTopic(SMnode *pMnode, SNodeMsg *pReq, SMqTopicObj *pTopic) { // TODO: cannot drop when subscribed STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { @@ -404,8 +404,8 @@ static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic return 0; } -static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropTopicReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SMDropTopicReq dropReq = {0}; if (tDeserializeSMDropTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { @@ -439,7 +439,7 @@ static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -471,8 +471,8 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo return 0; } -static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetTopicMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfTopics(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -515,8 +515,8 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqTopicObj *pTopic = NULL; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a009e01a52cfd1a535529cd71c47235b000a828a..33142114376c21b6ca4cd435c6ab3ea82281e370 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -55,11 +55,11 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); static void mndTransExecute(SMnode *pMnode, STrans *pTrans); static void mndTransSendRpcRsp(STrans *pTrans); -static int32_t mndProcessTransReq(SMnodeMsg *pReq); -static int32_t mndProcessKillTransReq(SMnodeMsg *pReq); +static int32_t mndProcessTransReq(SNodeMsg *pReq); +static int32_t mndProcessKillTransReq(SNodeMsg *pReq); -static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveTrans(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetTransMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); int32_t mndInitTrans(SMnode *pMnode) { @@ -255,7 +255,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < redoLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -265,7 +265,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < undoLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -275,7 +275,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < commitLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -288,7 +288,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) - action.pCont = malloc(action.contLen); + action.pCont = taosMemoryMalloc(action.contLen); if (action.pCont == NULL) goto TRANS_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto TRANS_DECODE_OVER; @@ -300,7 +300,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) - action.pCont = malloc(action.contLen); + action.pCont = taosMemoryMalloc(action.contLen); if (action.pCont == NULL) goto TRANS_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto TRANS_DECODE_OVER; @@ -315,9 +315,9 @@ TRANS_DECODE_OVER: if (terrno != 0) { mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); - tfree(pRow); - tfree(pData); - tfree(action.pCont); + taosMemoryFreeClear(pRow); + taosMemoryFreeClear(pData); + taosMemoryFreeClear(action.pCont); return NULL; } @@ -406,6 +406,10 @@ static const char *mndTransType(ETrnType type) { return "alter-stb"; case TRN_TYPE_DROP_STB: return "drop-stb"; + case TRN_TYPE_CREATE_SMA: + return "create-sma"; + case TRN_TYPE_DROP_SMA: + return "drop-sma"; default: return "invalid"; } @@ -424,7 +428,7 @@ static void mndTransDropData(STrans *pTrans) { mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->undoActions); if (pTrans->rpcRsp != NULL) { - free(pTrans->rpcRsp); + taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; } @@ -468,7 +472,7 @@ static void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { } STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq) { - STrans *pTrans = calloc(1, sizeof(STrans)); + STrans *pTrans = taosMemoryCalloc(1, sizeof(STrans)); if (pTrans == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to create transaction since %s", terrstr()); @@ -513,7 +517,7 @@ static void mndTransDropActions(SArray *pArray) { int32_t size = taosArrayGetSize(pArray); for (int32_t i = 0; i < size; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - tfree(pAction->pCont); + taosMemoryFreeClear(pAction->pCont); } taosArrayDestroy(pArray); @@ -523,7 +527,7 @@ void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); - tfree(pTrans); + taosMemoryFreeClear(pTrans); } } @@ -758,9 +762,9 @@ static void mndTransSendRpcRsp(STrans *pTrans) { if (rpcCont != NULL) { memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); } - free(pTrans->rpcRsp); + taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, @@ -774,8 +778,8 @@ static void mndTransSendRpcRsp(STrans *pTrans) { } } -void mndTransProcessRsp(SMnodeMsg *pRsp) { - SMnode *pMnode = pRsp->pMnode; +void mndTransProcessRsp(SNodeMsg *pRsp) { + SMnode *pMnode = pRsp->pNode; int64_t signature = (int64_t)(pRsp->rpcMsg.ahandle); int32_t transId = (int32_t)(signature >> 32); int32_t action = (int32_t)((signature << 32) >> 32); @@ -816,7 +820,7 @@ void mndTransProcessRsp(SMnodeMsg *pRsp) { } } - mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%x", transId, action, pRsp->rpcMsg.code, + mDebug("trans:%d, action:%d response is received, code:0x%04x, accept:0x%04x", transId, action, pRsp->rpcMsg.code, pAction->acceptableCode); mndTransExecute(pMnode, pTrans); @@ -888,7 +892,7 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr } memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); - if (mndSendReqToDnode(pMnode, &pAction->epSet, &rpcMsg) == 0) { + if (tmsgSendReq(&pMnode->msgCb, &pAction->epSet, &rpcMsg) == 0) { mDebug("trans:%d, action:%d is sent", pTrans->id, action); pAction->msgSent = 1; pAction->msgReceived = 0; @@ -928,13 +932,13 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); return 0; } else { - mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode); + mError("trans:%d, all %d actions executed, code:0x%04x", pTrans->id, numOfActions, errCode & 0XFFFF); mndTransResetActions(pMnode, pTrans, pArray); terrno = errCode; return errCode; } } else { - mDebug("trans:%d, %d of %d actions executed, code:0x%x", pTrans->id, numOfReceived, numOfActions, errCode); + mDebug("trans:%d, %d of %d actions executed, code:0x%04x", pTrans->id, numOfReceived, numOfActions, errCode & 0XFFFF); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } } @@ -1111,7 +1115,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); } - mDebug("trans:%d, finished, code:0x%x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); + mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); return continueExec; } @@ -1157,8 +1161,8 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { mndTransSendRpcRsp(pTrans); } -static int32_t mndProcessTransReq(SMnodeMsg *pReq) { - mndTransPullup(pReq->pMnode); +static int32_t mndProcessTransReq(SNodeMsg *pReq) { + mndTransPullup(pReq->pNode); return 0; } @@ -1199,8 +1203,8 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { return 0; } -static int32_t mndProcessKillTransReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SKillTransReq killReq = {0}; int32_t code = -1; SUserObj *pUser = NULL; @@ -1257,8 +1261,8 @@ void mndTransPullup(SMnode *pMnode) { sdbWriteFile(pMnode->pSdb); } -static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetTransMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -1320,8 +1324,8 @@ static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveTrans(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; STrans *pTrans = NULL; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b1cdf484b4805a9355da48db15cf5df5ca7ea91b..ff34c26c4acfb4c0f887187968e4dc0d8cf417a2 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -30,13 +30,13 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew); -static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SMnodeMsg *pReq); -static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq); -static int32_t mndProcessDropUserReq(SMnodeMsg *pReq); -static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq); -static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SNodeMsg *pReq); +static int32_t mndProcessCreateUserReq(SNodeMsg *pReq); +static int32_t mndProcessAlterUserReq(SNodeMsg *pReq); +static int32_t mndProcessDropUserReq(SNodeMsg *pReq); +static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq); +static int32_t mndGetUserMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); int32_t mndInitUser(SMnode *pMnode) { @@ -201,7 +201,7 @@ USER_DECODE_OVER: mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr()); taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -261,7 +261,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { sdbRelease(pSdb, pUser); } -static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SMnodeMsg *pReq) { +static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SNodeMsg *pReq) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)pCreate->pass, strlen(pCreate->pass), userObj.pass); tstrncpy(userObj.user, pCreate->user, TSDB_USER_LEN); @@ -295,8 +295,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate return 0; } -static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -349,7 +349,7 @@ CREATE_USER_OVER: return code; } -static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SMnodeMsg *pReq) { +static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER,&pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to update since %s", pOld->user, terrstr()); @@ -397,8 +397,8 @@ static SHashObj *mndDupDbHash(SHashObj *pOld) { return pNew; } -static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -510,7 +510,7 @@ ALTER_USER_OVER: return code; } -static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pReq, SUserObj *pUser) { +static int32_t mndDropUser(SMnode *pMnode, SNodeMsg *pReq, SUserObj *pUser) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); @@ -536,8 +536,8 @@ static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pReq, SUserObj *pUser) { return 0; } -static int32_t mndProcessDropUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -585,8 +585,8 @@ DROP_USER_OVER: return code; } -static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SGetUserAuthReq authReq = {0}; @@ -635,8 +635,8 @@ static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; code = 0; GET_AUTH_OVER: @@ -647,8 +647,8 @@ GET_AUTH_OVER: return code; } -static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetUserMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -693,8 +693,8 @@ static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p return 0; } -static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SUserObj *pUser = NULL; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index f7b177f170daa28f499d7b8a8d9fd22fcc98562c..d36617822da4fa4deeb496c0e842eace5f59d494 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -29,17 +29,17 @@ static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pRsp); - -static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessSyncVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); + +static int32_t mndGetVgroupMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetVnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { @@ -146,7 +146,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { VG_DECODE_OVER: if (terrno != 0) { mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -218,6 +218,8 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.hashBegin = pVgroup->hashBegin; createReq.hashEnd = pVgroup->hashEnd; createReq.hashMethod = pDb->hashMethod; + createReq.numOfRetensions = pDb->cfg.numOfRetensions; + createReq.pRetensions = pDb->cfg.pRetensions; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; @@ -248,7 +250,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return NULL; } - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -272,7 +274,7 @@ void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgOb return NULL; } - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -374,7 +376,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { SArray *pArray = NULL; SVgObj *pVgroups = NULL; - pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); + pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto ALLOC_VGROUP_OVER; @@ -428,17 +430,17 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); ALLOC_VGROUP_OVER: - if (code != 0) free(pVgroups); + if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); return code; } -SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { +SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet epset = {0}; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + const SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode == NULL) continue; if (pVgid->role == TAOS_SYNC_STATE_LEADER) { @@ -452,24 +454,24 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { return epset; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pRsp) { return 0; } +static int32_t mndProcessSyncVnodeRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pRsp) { return 0; } +static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp) { return 0; } static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; @@ -500,8 +502,8 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetVgroupMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetVgroupMaxReplica(pMnode, pShow->db, &pShow->replica, &pShow->numOfRows) != 0) { @@ -551,49 +553,79 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; int32_t cols = 0; char *pWrite; - SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) return 0; + SDbObj *pDb = NULL; + if (strlen(pShow->db) > 0) { + pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) { + return 0; + } + } while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); if (pShow->pIter == NULL) break; - if (pVgroup->dbUid == pDb->uid) { - cols = 0; + if (pDb != NULL && pVgroup->dbUid != pDb->uid) { + continue; + } + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pVgroup->vgId; + cols++; + + SName name = {0}; + char db[TSDB_DB_NAME_LEN] = {0}; + tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT|T_NAME_DB); + tNameGetDbName(&name, db); + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, db); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pVgroup->numOfTables; + cols++; + + //status + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, "ready"); // TODO + cols++; + + //onlines + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pVgroup->replica; + cols++; + + for (int32_t i = 0; i < pVgroup->replica; ++i) { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pVgroup->vgId; + *(int16_t *)pWrite = pVgroup->vnodeGid[i].dnodeId; cols++; + const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pVgroup->numOfTables; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); cols++; - - for (int32_t i = 0; i < pShow->replica; ++i) { - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pVgroup->vnodeGid[i].dnodeId; - cols++; - - const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); - cols++; - } - numOfRows++; } + numOfRows++; sdbRelease(pSdb, pVgroup); } - mndReleaseDb(pMnode, pDb); + if (pDb != NULL) { + mndReleaseDb(pMnode, pDb); + } + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; @@ -624,8 +656,8 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetVnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -664,14 +696,14 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; char *pWrite; int32_t cols = 0; - int32_t dnodeId = pShow->replica; +// int32_t dnodeId = pShow->replica; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); @@ -679,17 +711,33 @@ static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, i for (int32_t i = 0; i < pVgroup->replica && numOfRows < rows; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - if (pVgid->dnodeId != dnodeId) continue; - cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(uint32_t *)pWrite = pVgroup->vgId; cols++; + SName name = {0}; + char db[TSDB_DB_NAME_LEN] = {0}; + tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT|T_NAME_DB); + tNameGetDbName(&name, db); + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, db); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(uint32_t *)pWrite = 0; //todo: Tables + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_TO_VARSTR(pWrite, mndGetRoleStr(pVgid->role)); cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(uint32_t *)pWrite = pVgroup->replica; //onlines + cols++; + numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index f2baea1cd9577b6a7592eac4edc976c4179d8d2c..67b7d6dd45f754a6378c3b25941b5118ce62f61c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -28,6 +28,7 @@ #include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" +#include "mndSma.h" #include "mndSnode.h" #include "mndStb.h" #include "mndStream.h" @@ -38,35 +39,12 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndQuery.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 #define TELEM_TIMER_MS 86400000 -int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { - if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { - terrno = TSDB_CODE_MND_NOT_READY; - return -1; - } - - return (*pMnode->sendReqToDnodeFp)(pMnode->pDnode, pEpSet, pMsg); -} - -int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg) { - if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { - terrno = TSDB_CODE_MND_NOT_READY; - return -1; - } - - return (*pMnode->sendReqToMnodeFp)(pMnode->pDnode, pMsg); -} - -void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg) { - if (pMnode != NULL && pMnode->sendRedirectRspFp != NULL) { - (*pMnode->sendRedirectRspFp)(pMnode->pDnode, pMsg); - } -} - static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -86,7 +64,7 @@ static void mndPullupTrans(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; - pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); @@ -102,7 +80,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { .pCont = pReq, .contLen = contLen, }; - pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); @@ -114,7 +92,7 @@ static void mndPullupTelem(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; - pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); @@ -228,6 +206,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-stb", mndInitSma, mndCleanupSma) != 0) return -1; if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; @@ -236,9 +215,10 @@ static int32_t mndInitSteps(SMnode *pMnode) { } else { if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1; } - if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; + // if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; if (mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem) != 0) return -1; if (mndAllocStep(pMnode, "mnode-timer", NULL, mndCleanupTimer) != 0) return -1; @@ -292,15 +272,9 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->replica = pOption->replica; pMnode->selfIndex = pOption->selfIndex; memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); - pMnode->pDnode = pOption->pDnode; - pMnode->putReqToMWriteQFp = pOption->putReqToMWriteQFp; - pMnode->putReqToMReadQFp = pOption->putReqToMReadQFp; - pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp; - pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp; - pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp; - - if (pMnode->sendReqToDnodeFp == NULL || pMnode->sendReqToMnodeFp == NULL || pMnode->sendRedirectRspFp == NULL || - pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { + pMnode->msgCb = pOption->msgCb; + + if (pMnode->dnodeId < 0 || pMnode->clusterId < 0) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } @@ -311,7 +285,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { mDebug("start to open mnode in %s", path); - SMnode *pMnode = calloc(1, sizeof(SMnode)); + SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode)); if (pMnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to open mnode since %s", terrstr()); @@ -323,7 +297,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep)); if (pMnode->pSteps == NULL) { - free(pMnode); + taosMemoryFree(pMnode); terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to open mnode since %s", terrstr()); return NULL; @@ -374,8 +348,8 @@ void mndClose(SMnode *pMnode) { if (pMnode != NULL) { mDebug("start to close mnode"); mndCleanupSteps(pMnode, -1); - tfree(pMnode->path); - tfree(pMnode); + taosMemoryFreeClear(pMnode->path); + taosMemoryFreeClear(pMnode); mDebug("mnode is closed"); } } @@ -386,108 +360,47 @@ int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption) { return 0; } -void mndDestroy(const char *path) { - mDebug("start to destroy mnode at %s", path); - taosRemoveDir(path); - mDebug("mnode is destroyed"); -} +int32_t mndStart(SMnode *pMnode) { return mndInitTimer(pMnode); } -SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { - SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("failed to create msg since %s, app:%p RPC:%p", terrstr(), pRpcMsg->ahandle, pRpcMsg->handle); - return NULL; - } +int32_t mndProcessMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SRpcMsg *pRpc = &pMsg->rpcMsg; + tmsg_t msgType = pMsg->rpcMsg.msgType; + void *ahandle = pMsg->rpcMsg.ahandle; + bool isReq = (pRpc->msgType & 1U); - if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && - pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) { - SRpcConnInfo connInfo = {0}; - if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { - taosFreeQitem(pMsg); - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - mError("failed to create msg since %s, app:%p RPC:%p", terrstr(), pRpcMsg->ahandle, pRpcMsg->handle); - return NULL; - } - memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); - } - - pMsg->pMnode = pMnode; - pMsg->rpcMsg = *pRpcMsg; - pMsg->createdTime = taosGetTimestampSec(); - - if (pRpcMsg != NULL) { - mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); - } - return pMsg; -} - -void mndCleanupMsg(SMnodeMsg *pMsg) { - mTrace("msg:%p, is destroyed", pMsg); - rpcFreeCont(pMsg->rpcMsg.pCont); - pMsg->rpcMsg.pCont = NULL; - taosFreeQitem(pMsg); -} - -void mndSendRsp(SMnodeMsg *pMsg, int32_t code) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .code = code}; - rpcSendResponse(&rpcRsp); -} - -void mndProcessMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - int32_t code = 0; - tmsg_t msgType = pMsg->rpcMsg.msgType; - void *ahandle = pMsg->rpcMsg.ahandle; - bool isReq = (msgType & 1U); - - mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle); + mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(msgType), ahandle); if (isReq && !mndIsMaster(pMnode)) { - code = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_NOT_READY; mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; + return -1; } - if (isReq && pMsg->rpcMsg.pCont == NULL) { - code = TSDB_CODE_MND_INVALID_MSG_LEN; + if (isReq && (pRpc->contLen == 0 || pRpc->pCont == NULL)) { + terrno = TSDB_CODE_MND_INVALID_MSG_LEN; mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; + return -1; } MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(msgType)]; if (fp == NULL) { - code = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("msg:%p, failed to process since no msg handle, app:%p", pMsg, ahandle); - goto PROCESS_RPC_END; + return -1; } - code = (*fp)(pMsg); + int32_t code = (*fp)(pMsg); if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + terrno = code; mTrace("msg:%p, in progress, app:%p", pMsg, ahandle); - return; } else if (code != 0) { - code = terrno; mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; } else { mTrace("msg:%p, is processed, app:%p", pMsg, ahandle); } -PROCESS_RPC_END: - if (isReq) { - if (pMsg->rpcMsg.handle == NULL) return; - - if (code == TSDB_CODE_APP_NOT_READY) { - mndSendRedirectRsp(pMnode, &pMsg->rpcMsg); - } else if (code != 0) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont, .code = code}; - rpcSendResponse(&rpcRsp); - } else { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont}; - rpcSendResponse(&rpcRsp); - } - } + return code; } void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { @@ -498,15 +411,15 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { } // Note: uid 0 is reserved -uint64_t mndGenerateUid(char *name, int32_t len) { +int64_t mndGenerateUid(char *name, int32_t len) { int32_t hashval = MurmurHash3_32(name, len); do { - int64_t us = taosGetTimestampUs(); - uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; - uint64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + int64_t us = taosGetTimestampUs(); + int64_t x = (us & 0x000000FFFFFFFFFF) << 24; + int64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); if (uuid) { - return uuid; + return llabs(uuid); } } while (true); } diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index df0510f78369fde33b45cb458337d7fe8ac33a06..61201f33c36bb762260e96e2b9e726f7a3317a52 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -12,5 +12,6 @@ add_subdirectory(dnode) add_subdirectory(mnode) add_subdirectory(db) add_subdirectory(stb) +add_subdirectory(sma) add_subdirectory(func) add_subdirectory(topic) diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index d7d15df35a7a3838594ea6927bc84faef1457043..4d691692ffac460494fb9f07f141e3e68b292718 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -317,4 +317,4 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { ASSERT_NE(retry, retryMax); } -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/db/CMakeLists.txt b/source/dnode/mnode/impl/test/db/CMakeLists.txt index f0abdf152cd5b081506a40d2f3b524560b47505c..d1e38854e6f0e7a181c0801bfd76b7249b03a226 100644 --- a/source/dnode/mnode/impl/test/db/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/db/CMakeLists.txt @@ -5,7 +5,7 @@ target_link_libraries( PUBLIC sut ) -add_test( - NAME mnode_test_db - COMMAND mnode_test_db -) +#add_test( +# NAME mnode_test_db +# COMMAND mnode_test_db +#) diff --git a/source/dnode/mnode/impl/test/dnode/CMakeLists.txt b/source/dnode/mnode/impl/test/dnode/CMakeLists.txt index e29c5e8f3d0798cc819455df865758063040df22..77cc6f0b5dbefe7dcf4c7de2e836188e19c51c13 100644 --- a/source/dnode/mnode/impl/test/dnode/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/dnode/CMakeLists.txt @@ -5,7 +5,7 @@ target_link_libraries( PUBLIC sut ) -add_test( - NAME mnode_test_dnode - COMMAND mnode_test_dnode -) +#add_test( +# NAME mnode_test_dnode +# COMMAND mnode_test_dnode +#) diff --git a/source/dnode/mnode/impl/test/sma/CMakeLists.txt b/source/dnode/mnode/impl/test/sma/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..943695abf3a1cc353b50e621d9d7975288b2616d --- /dev/null +++ b/source/dnode/mnode/impl/test/sma/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. SMA_SRC) +add_executable(mnode_test_sma ${SMA_SRC}) +target_link_libraries( + mnode_test_sma + PUBLIC sut +) + +add_test( + NAME mnode_test_sma + COMMAND mnode_test_sma +) diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp new file mode 100644 index 0000000000000000000000000000000000000000..85f6a86183e21d7c041ff4e18124f1a36a9e3037 --- /dev/null +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -0,0 +1,294 @@ +/** + * @file sma.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module sma tests + * @version 1.0 + * @date 2022-03-23 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class MndTestSma : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_sma", 9035); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} + + void* BuildCreateDbReq(const char* dbname, int32_t* pContLen); + void* BuildDropDbReq(const char* dbname, int32_t* pContLen); + void* BuildCreateStbReq(const char* stbname, int32_t* pContLen); + void* BuildDropStbReq(const char* stbname, int32_t* pContLen); + void* BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen); + void* BuildCreateTSmaReq(const char* smaname, const char* stbname, int8_t igExists, const char* expr, + const char* tagsFilter, const char* sql, const char* ast, int32_t* pContLen); + void* BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int32_t* pContLen); + + void PushField(SArray* pArray, int32_t bytes, int8_t type, const char* name); +}; + +Testbase MndTestSma::test; + +void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { + SCreateDbReq createReq = {0}; + strcpy(createReq.db, dbname); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestSma::BuildDropDbReq(const char* dbname, int32_t* pContLen) { + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, dbname); + + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); + + *pContLen = contLen; + return pReq; +} + +void MndTestSma::PushField(SArray* pArray, int32_t bytes, int8_t type, const char* name) { + SField field = {0}; + field.bytes = bytes; + field.type = type; + strcpy(field.name, name); + taosArrayPush(pArray, &field); +} + +void* MndTestSma::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { + SMCreateStbReq createReq = {0}; + createReq.numOfColumns = 3; + createReq.numOfTags = 1; + createReq.igExists = 0; + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + strcpy(createReq.name, stbname); + + PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts"); + PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1"); + PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2"); + PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1"); + + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateStbReq(pHead, tlen, &createReq); + tFreeSMCreateStbReq(&createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestSma::BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen) { + SMCreateStbReq createReq = {0}; + createReq.numOfColumns = 3; + createReq.numOfTags = 1; + createReq.numOfSmas = 1; + createReq.igExists = 0; + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + createReq.pSmas = taosArrayInit(createReq.numOfSmas, sizeof(SField)); + strcpy(createReq.name, stbname); + + PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts"); + PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1"); + PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2"); + PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1"); + PushField(createReq.pSmas, 2, TSDB_DATA_TYPE_TINYINT, "col1"); + + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateStbReq(pHead, tlen, &createReq); + tFreeSMCreateStbReq(&createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestSma::BuildDropStbReq(const char* stbname, int32_t* pContLen) { + SMDropStbReq dropstbReq = {0}; + strcpy(dropstbReq.name, stbname); + + int32_t contLen = tSerializeSMDropStbReq(NULL, 0, &dropstbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMDropStbReq(pReq, contLen, &dropstbReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestSma::BuildCreateTSmaReq(const char* smaname, const char* stbname, int8_t igExists, const char* expr, + const char* tagsFilter, const char* sql, const char* ast, int32_t* pContLen) { + SMCreateSmaReq createReq = {0}; + strcpy(createReq.name, smaname); + strcpy(createReq.stb, stbname); + createReq.igExists = igExists; + createReq.intervalUnit = 1; + createReq.slidingUnit = 2; + createReq.timezone = 3; + createReq.dstVgId = 4; + createReq.interval = 10; + createReq.offset = 5; + createReq.sliding = 6; + createReq.expr = (char*)expr; + createReq.exprLen = strlen(createReq.expr) + 1; + createReq.tagsFilter = (char*)tagsFilter; + createReq.tagsFilterLen = strlen(createReq.tagsFilter) + 1; + createReq.sql = (char*)sql; + createReq.sqlLen = strlen(createReq.sql) + 1; + createReq.ast = (char*)ast; + createReq.astLen = strlen(createReq.ast) + 1; + + int32_t tlen = tSerializeSMCreateSmaReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateSmaReq(pHead, tlen, &createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestSma::BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int32_t* pContLen) { + SMDropSmaReq dropsmaReq = {0}; + dropsmaReq.igNotExists = igNotExists; + strcpy(dropsmaReq.name, smaname); + + int32_t contLen = tSerializeSMDropSmaReq(NULL, 0, &dropsmaReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMDropSmaReq(pReq, contLen, &dropsmaReq); + + *pContLen = contLen; + return pReq; +} + +TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) { + #if 0 + const char* dbname = "1.d1"; + const char* stbname = "1.d1.stb"; + const char* smaname = "1.d1.sma"; + int32_t contLen = 0; + void* pReq; + SRpcMsg* pRsp; + + { + pReq = BuildCreateDbReq(dbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + pReq = BuildCreateStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } + + { + pReq = BuildCreateTSmaReq(smaname, stbname, 0, "expr", "tagsFilter", "sql", "ast", &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_SMA, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } + + // restart + test.Restart(); + + { + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + CHECK_META("show indexes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("sma", TSDB_INDEX_NAME_LEN); + CheckTimestamp(); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + } + + { + pReq = BuildDropTSmaReq(smaname, 0, &contLen); + pRsp = test.SendReq(TDMT_MND_DROP_SMA, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); + } +#endif +} + +TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { + const char* dbname = "1.d1"; + const char* stbname = "1.d1.bsmastb"; + int32_t contLen = 0; + void* pReq; + SRpcMsg* pRsp; + + { + pReq = BuildCreateDbReq(dbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + pReq = BuildCreateBSmaStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); +// CheckBinary("bsmastb", TSDB_TABLE_NAME_LEN); + } + + test.Restart(); + + { + pReq = BuildCreateBSmaStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_ALREADY_EXIST); + } + + { + pReq = BuildDropStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); + } + + { + pReq = BuildDropStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST); + } +} diff --git a/source/dnode/mnode/impl/test/stb/CMakeLists.txt b/source/dnode/mnode/impl/test/stb/CMakeLists.txt index 72197daf0997f00f5eeee006d16d8240c2741f48..eca922acc43da5f3fd737c8add16854c6a781e99 100644 --- a/source/dnode/mnode/impl/test/stb/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/stb/CMakeLists.txt @@ -5,7 +5,7 @@ target_link_libraries( PUBLIC sut ) -add_test( - NAME mnode_test_stb - COMMAND mnode_test_stb -) +#add_test( +# NAME mnode_test_stb +# COMMAND mnode_test_stb +#) diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index cea93017f4bbc1dd042d55a46a01561dfebd5881..b37ec85387ce7eb28ac9c2aeb6918c37b9397e67 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -29,7 +29,7 @@ class MndTestTrans : public ::testing::Test { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; - void* buffer = malloc(size); + void* buffer = taosMemoryMalloc(size); int32_t readLen = taosReadFile(pFile, buffer, size); if (readLen < 0 || readLen == size) { ASSERT(1); @@ -43,7 +43,7 @@ class MndTestTrans : public ::testing::Test { if (writeLen < 0 || writeLen == readLen) { ASSERT(1); } - free(buffer); + taosMemoryFree(buffer); taosFsyncFile(pFile); taosCloseFile(&pFile); taosMsleep(1000); @@ -204,6 +204,8 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } + taosMsleep(1000); + { // show trans test.SendShowMetaReq(TSDB_MGMT_TABLE_TRANS, ""); @@ -241,6 +243,7 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { EXPECT_EQ(test.GetShowRows(), 0); } + uInfo("======== re-create trans"); // re-create trans { SMCreateQnodeReq createReq = {0}; @@ -255,10 +258,14 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } + uInfo("======== kill and restart server") KillThenRestartServer(); + uInfo("======== server2 start") server2.DoStart(); + uInfo("======== server2 started") + { int32_t retry = 0; int32_t retryMax = 20; diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 26809ea0596c1f339c4135938094606af0cfd567..c645dae5b5e68e6dd65ce450f30d3162864fb6a5 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -21,7 +21,7 @@ static int32_t sdbCreateDir(SSdb *pSdb); SSdb *sdbInit(SSdbOpt *pOption) { mDebug("start to init sdb in %s", pOption->path); - SSdb *pSdb = calloc(1, sizeof(SSdb)); + SSdb *pSdb = taosMemoryCalloc(1, sizeof(SSdb)); if (pSdb == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to init sdb since %s", terrstr()); @@ -67,15 +67,15 @@ void sdbCleanup(SSdb *pSdb) { sdbWriteFile(pSdb); if (pSdb->currDir != NULL) { - tfree(pSdb->currDir); + taosMemoryFreeClear(pSdb->currDir); } if (pSdb->syncDir != NULL) { - tfree(pSdb->syncDir); + taosMemoryFreeClear(pSdb->syncDir); } if (pSdb->tmpDir != NULL) { - tfree(pSdb->tmpDir); + taosMemoryFreeClear(pSdb->tmpDir); } for (ESdbType i = 0; i < SDB_MAX; ++i) { @@ -102,7 +102,7 @@ void sdbCleanup(SSdb *pSdb) { mDebug("sdb table:%d is cleaned up", i); } - free(pSdb); + taosMemoryFree(pSdb); mDebug("sdb is cleaned up"); } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 2849da5c2ec1910c42fd62a95c4eb1629c56ad71..c11025beedfa8cbb5d8e032a28cef26319ae6d9a 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -137,7 +137,7 @@ int32_t sdbReadFile(SSdb *pSdb) { int32_t readLen = 0; int64_t ret = 0; - SSdbRaw *pRaw = malloc(SDB_MAX_SIZE); + SSdbRaw *pRaw = taosMemoryMalloc(SDB_MAX_SIZE); if (pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed read file since %s", terrstr()); @@ -150,7 +150,7 @@ int32_t sdbReadFile(SSdb *pSdb) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - free(pRaw); + taosMemoryFree(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, terrstr()); return 0; @@ -159,7 +159,7 @@ int32_t sdbReadFile(SSdb *pSdb) { if (sdbReadFileHead(pSdb, pFile) != 0) { mError("failed to read file:%s head since %s", file, terrstr()); pSdb->curVer = -1; - free(pRaw); + taosMemoryFree(pRaw); taosCloseFile(&pFile); return -1; } diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index e37559808e2b5242354be6187c07fc49a732a940..c3aaf562bef5a557762651e770b049ee3b949416 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -17,7 +17,7 @@ #include "sdbInt.h" SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { - SSdbRaw *pRaw = calloc(1, dataLen + sizeof(SSdbRaw)); + SSdbRaw *pRaw = taosMemoryCalloc(1, dataLen + sizeof(SSdbRaw)); if (pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -33,7 +33,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { void sdbFreeRaw(SSdbRaw *pRaw) { mTrace("raw:%p, is freed", pRaw); - free(pRaw); + taosMemoryFree(pRaw); } int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { diff --git a/source/dnode/mnode/sdb/src/sdbRow.c b/source/dnode/mnode/sdb/src/sdbRow.c index e51dd48dcdd9fff0652e13d5b94cbe961e93bfad..ac86a72155f1bd178ab6a83b8a06d0198ad31cf2 100644 --- a/source/dnode/mnode/sdb/src/sdbRow.c +++ b/source/dnode/mnode/sdb/src/sdbRow.c @@ -17,7 +17,7 @@ #include "sdbInt.h" SSdbRow *sdbAllocRow(int32_t objSize) { - SSdbRow *pRow = calloc(1, objSize + sizeof(SSdbRow)); + SSdbRow *pRow = taosMemoryCalloc(1, objSize + sizeof(SSdbRow)); if (pRow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -46,5 +46,5 @@ void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) { sdbPrintOper(pSdb, pRow, "freeRow"); mTrace("row:%p, is freed", pRow->pObj); - tfree(pRow); + taosMemoryFreeClear(pRow); } diff --git a/source/dnode/qnode/inc/qndInt.h b/source/dnode/qnode/inc/qndInt.h index 357a62052aea7c096c5ac87465c126af9de8338f..307bf3efb8913d4fae4966797835e432ebb051c5 100644 --- a/source/dnode/qnode/inc/qndInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -21,6 +21,7 @@ #include "tlog.h" #include "tmsg.h" #include "trpc.h" +#include "tmsgcb.h" #include "qnode.h" @@ -32,8 +33,8 @@ typedef struct SQWorkerMgmt SQHandle; typedef struct SQnode { int32_t qndId; - SQnodeOpt opt; - SQHandle* pQuery; + SMsgCb msgCb; + SQHandle* pQuery; } SQnode; #ifdef __cplusplus diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index a1c3f5b0d42925d86d28a7e21dfd5dcb4cd8ae96..0d1e890097e93eef19a1500eba0794ca0452d57c 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -13,50 +13,41 @@ * along with this program. If not, see . */ +#include "executor.h" #include "qndInt.h" #include "query.h" #include "qworker.h" -#include "executor.h" - -int32_t qnodePutReqToVQueryQ(SQnode* pQnode, struct SRpcMsg* pReq) {} -void qnodeSendReqToDnode(SQnode* pQnode, struct SEpSet* epSet, struct SRpcMsg* pReq) {} - SQnode *qndOpen(const SQnodeOpt *pOption) { - SQnode *pQnode = calloc(1, sizeof(SQnode)); + SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); if (NULL == pQnode) { qError("calloc SQnode failed"); return NULL; } - if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, pQnode, - (putReqToQueryQFp)qnodePutReqToVQueryQ, (sendReqToDnodeFp)qnodeSendReqToDnode)) { - tfree(pQnode); + if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { + taosMemoryFreeClear(pQnode); return NULL; } - + + pQnode->msgCb = pOption->msgCb; return pQnode; } -void qndClose(SQnode *pQnode) { +void qndClose(SQnode *pQnode) { qWorkerDestroy((void **)&pQnode->pQuery); - free(pQnode); + taosMemoryFree(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } -int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - *pRsp = NULL; - return 0; -} - -int qnodeProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { +int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { qTrace("message in query queue is processing"); SReadHandle handle = {0}; switch (pMsg->msgType) { - case TDMT_VND_QUERY:{ + case TDMT_VND_QUERY: { return qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg); } case TDMT_VND_QUERY_CONTINUE: @@ -67,7 +58,7 @@ int qnodeProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { } } -int qnodeProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { +int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { qTrace("message in fetch queue is processing"); switch (pMsg->msgType) { case TDMT_VND_FETCH: @@ -85,17 +76,13 @@ int qnodeProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { case TDMT_VND_SHOW_TABLES: return qWorkerProcessShowMsg(pQnode, pQnode->pQuery, pMsg); case TDMT_VND_SHOW_TABLES_FETCH: - //return vnodeGetTableList(pQnode, pMsg); + // return vnodeGetTableList(pQnode, pMsg); case TDMT_VND_TABLE_META: - //return vnodeGetTableMeta(pQnode, pMsg); + // return vnodeGetTableMeta(pQnode, pMsg); case TDMT_VND_CONSUME: - //return tqProcessConsumeReq(pQnode->pTq, pMsg); + // return tqProcessConsumeReq(pQnode->pTq, pMsg); default: qError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; } } - - - - diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index d1b1abdf1d523aca7fd0ccb56180b7535fde40fd..f177bda47aa12a6f7899a1fda8b09a56bf3046d5 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -12,4 +12,6 @@ target_link_libraries( PRIVATE os PRIVATE common PRIVATE util + PRIVATE qcom + PRIVATE stream ) diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index d1122fc4ecaae129ada2e427bb223cb6780e8e44..2802537dcd31323a703cd6a4d2222bb8fd9d3d68 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -22,6 +22,7 @@ #include "tmsg.h" #include "tqueue.h" #include "trpc.h" +#include "tstream.h" #include "snode.h" @@ -44,7 +45,7 @@ typedef struct { typedef struct SSnode { SStreamMeta* pMeta; - SSnodeOpt cfg; + SMsgCb msgCb; } SSnode; SStreamMeta* sndMetaNew(); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 2ecaeb00e90ee0d9e07e69c4e42abac360529f94..a5937b625f11e449dea7e76cf0397e96c503f439 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -18,14 +18,14 @@ #include "tuuid.h" SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { - SSnode *pSnode = calloc(1, sizeof(SSnode)); + SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); if (pSnode == NULL) { return NULL; } - memcpy(&pSnode->cfg, pOption, sizeof(SSnodeOpt)); + pSnode->msgCb = pOption->msgCb; pSnode->pMeta = sndMetaNew(); if (pSnode->pMeta == NULL) { - free(pSnode); + taosMemoryFree(pSnode); return NULL; } return pSnode; @@ -33,26 +33,19 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { void sndClose(SSnode *pSnode) { sndMetaDelete(pSnode->pMeta); - free(pSnode); + taosMemoryFree(pSnode); } int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } -/*int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {*/ -/**pRsp = NULL;*/ -/*return 0;*/ -/*}*/ - -void sndDestroy(const char *path) {} - SStreamMeta *sndMetaNew() { - SStreamMeta *pMeta = calloc(1, sizeof(SStreamMeta)); + SStreamMeta *pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { return NULL; } pMeta->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pMeta->pHash == NULL) { - free(pMeta); + taosMemoryFree(pMeta); return NULL; } return pMeta; @@ -60,11 +53,13 @@ SStreamMeta *sndMetaNew() { void sndMetaDelete(SStreamMeta *pMeta) { taosHashCleanup(pMeta->pHash); - free(pMeta); + taosMemoryFree(pMeta); } int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) { - pTask->executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL); + for (int i = 0; i < pTask->exec.numOfRunners; i++) { + pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); + } return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *)); } @@ -77,31 +72,32 @@ int32_t sndMetaRemoveTask(SStreamMeta *pMeta, int32_t taskId) { if (pTask == NULL) { return -1; } - free(pTask->qmsg); + taosMemoryFree(pTask->exec.qmsg); // TODO:free executor - free(pTask); + taosMemoryFree(pTask); return taosHashRemove(pMeta->pHash, &taskId, sizeof(int32_t)); } static int32_t sndProcessTaskExecReq(SSnode *pSnode, SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - int32_t taskId = pHead->streamTaskId; - SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId); - if (pTask == NULL) { - return -1; - } + /*SStreamExecMsgHead *pHead = pMsg->pCont;*/ + /*int32_t taskId = pHead->streamTaskId;*/ + /*SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId);*/ + /*if (pTask == NULL) {*/ + /*return -1;*/ + /*}*/ return 0; } -int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { +void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { // stream deploy // stream stop/resume // operator exec if (pMsg->msgType == TDMT_SND_TASK_DEPLOY) { void *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - SStreamTask *pTask = malloc(sizeof(SStreamTask)); + SStreamTask *pTask = taosMemoryMalloc(sizeof(SStreamTask)); if (pTask == NULL) { - return -1; + ASSERT(0); + return; } SCoder decoder; tCoderInit(&decoder, TD_LITTLE_ENDIAN, msg, pMsg->contLen - sizeof(SMsgHead), TD_DECODER); @@ -114,15 +110,13 @@ int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { } else { ASSERT(0); } - return 0; } -int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { +void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { // operator exec if (pMsg->msgType == TDMT_SND_TASK_EXEC) { sndProcessTaskExecReq(pSnode, pMsg); } else { ASSERT(0); } - return 0; } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index f026f8331b9f8cb32d14dbb49a3f130ee6bf23cd..bdc8a71b0483b390f4284064071fe267ba066378 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -43,6 +43,7 @@ target_link_libraries( PUBLIC wal PUBLIC scheduler PUBLIC executor + PUBLIC stream PUBLIC qworker PUBLIC sync ) diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index 05749884d3107f8c5079b75313435fc4585ac029..149aac1206344f0526edcfe0f7451525ea96845e 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -51,7 +51,7 @@ int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg); -int32_t metaDropTSma(SMeta *pMeta, char *indexName); +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid); // For Query STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); @@ -61,6 +61,7 @@ STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); STSma * metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid); STSmaWrapper * metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid); SArray * metaGetSmaTbUids(SMeta *pMeta, bool isDup); +int metaGetTbNum(SMeta *pMeta); SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h deleted file mode 100644 index 8d8ed2e427087798c7e6da03be47743cec526bfa..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/inc/tq.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TQ_H_ -#define _TQ_H_ - -#include "executor.h" -#include "meta.h" -#include "taoserror.h" -#include "tcommon.h" -#include "tmallocator.h" -#include "tmsg.h" -#include "trpc.h" -#include "ttimer.h" -#include "tutil.h" -#include "vnode.h" -#include "wal.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct STQ STQ; - -// memory allocator provided by vnode -typedef struct { - SMemAllocatorFactory* pAllocatorFactory; - SMemAllocator* pAllocator; -} STqMemRef; - -// init once -int tqInit(); -void tqCleanUp(); - -// open in each vnode -STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); -void tqClose(STQ*); - -// required by vnode -int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version); -int tqCommit(STQ*); - -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessSetConnReq(STQ* pTq, char* msg); -int32_t tqProcessRebReq(STQ* pTq, char* msg); -int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); - -#ifdef __cplusplus -} -#endif - -#endif /*_TQ_H_*/ diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 7b93f7580d02465fe16ddf362aa558d17121e6a4..f1083c0d918d9a158cc5013f6b0eaa993590ac62 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -87,6 +87,15 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); int tsdbPrepareCommit(STsdb *pTsdb); int tsdbCommit(STsdb *pTsdb); +/** + * @brief When submit msg received, update the relative expired window synchronously. + * + * @param pTsdb + * @param msg + * @return int32_t + */ +int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, const char *msg); + /** * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine * @@ -95,10 +104,18 @@ int tsdbCommit(STsdb *pTsdb); * @return int32_t */ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg); -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); /** - * @brief Insert RSma(Time-range-wise Rollup SMA) data. + * @brief Drop tSma data and local cache. + * + * @param pTsdb + * @param indexUid + * @return int32_t + */ +int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); + +/** + * @brief Insert RSma(Rollup SMA) data. * * @param pTsdb * @param msg @@ -107,6 +124,20 @@ int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); // TODO: This is the basic params, and should wrap the params to a queryHandle. +/** + * @brief Get tSma(Time-range-wise SMA) data. + * + * @param pTsdb + * @param pData + * @param indexUid + * @param interval + * @param intervalUnit + * @param tableUid + * @param colId + * @param querySKey + * @param nMaxResult + * @return int32_t + */ int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult); @@ -245,6 +276,8 @@ int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGro */ void tsdbCleanupReadHandle(tsdbReaderT queryHandle); +int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b6dd90a4e2586f44a3865e73ef335be92e778919..dd4b8d84ca1a7a39a949d384b9bc2253a71b5d43 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -18,6 +18,7 @@ #include "os.h" #include "trpc.h" +#include "tmsgcb.h" #include "meta.h" #include "tarray.h" @@ -30,11 +31,8 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SVnode SVnode; -typedef struct SDnode SDnode; -typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); - +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SVnode SVnode; typedef struct { // TODO int32_t reserved; @@ -43,7 +41,6 @@ typedef struct { typedef struct { int32_t vgId; uint64_t dbId; - SDnode *pDnode; STfs *pTfs; uint64_t wsize; uint64_t ssize; @@ -57,17 +54,12 @@ typedef struct { SMetaCfg metaCfg; STqCfg tqCfg; SWalCfg walCfg; + SMsgCb msgCb; uint32_t hashBegin; uint32_t hashEnd; int8_t hashMethod; } SVnodeCfg; -typedef struct { - uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) - PutReqToVQueryQFp putReqToVQueryQFp; - SendReqToDnodeFp sendReqToDnodeFp; -} SVnodeOpt; - typedef struct { int64_t ver; int64_t tbUid; @@ -87,10 +79,9 @@ typedef struct { /** * @brief Initialize the vnode module * - * @param pOption Option of the vnode mnodule * @return int 0 for success and -1 for failure */ -int vnodeInit(const SVnodeOpt *pOption); +int vnodeInit(); /** * @brief Cleanup the vnode module @@ -126,9 +117,8 @@ void vnodeDestroy(const char *path); * * @param pVnode The vnode object. * @param pMsgs The array of SRpcMsg - * @return int 0 for success, -1 for failure */ -int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs); +void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs); /** * @brief Apply a write request message. @@ -193,6 +183,9 @@ void vnodeOptionsInit(SVnodeCfg *pOptions); */ void vnodeOptionsClear(SVnodeCfg *pOptions); +int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName); + + /* ------------------------ FOR COMPILE ------------------------ */ int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg); diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h index 16a53baef073813a707d12071c1138518a403af6..bc1017f0c793ced80165c8adb964a153e583acea 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/metaDef.h @@ -34,7 +34,7 @@ void metaCloseDB(SMeta* pMeta); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); -int metaRemoveSmaFromDb(SMeta* pMeta, const char* indexName); +int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); // SMetaCache int metaOpenCache(SMeta* pMeta); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 9b21cf92ce83a161e56031cf8569f1337f3a180c..f0c3f6801a189acd7b06039f9474a7527c895931 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -18,8 +18,8 @@ #include "meta.h" #include "tlog.h" -#include "tq.h" #include "tqPush.h" +#include "vnd.h" #ifdef __cplusplus extern "C" { @@ -153,17 +153,24 @@ typedef struct { FTqDelete pDeleter; } STqMetaStore; +typedef struct { + SMemAllocatorFactory* pAllocatorFactory; + SMemAllocator* pAllocator; +} STqMemRef; + struct STQ { // the collection of groups // the handle of meta kvstore + bool writeTrigger; char* path; STqCfg* tqConfig; STqMemRef tqMemRef; STqMetaStore* tqMeta; - STqPushMgr* tqPushMgr; - SHashObj* pStreamTasks; - SWal* pWal; - SMeta* pVnodeMeta; + // STqPushMgr* tqPushMgr; + SHashObj* pStreamTasks; + SVnode* pVnode; + SWal* pWal; + SMeta* pVnodeMeta; }; typedef struct { diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 5e4c85262154b1503cd3b5a1cbce39e09657e0c4..02aba95517ceb6f639b91586150692b598c62b3e 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -16,6 +16,7 @@ #ifndef _TD_TSDB_DEF_H_ #define _TD_TSDB_DEF_H_ +#include "tsdbDBDef.h" #include "tmallocator.h" #include "meta.h" #include "tcompression.h" @@ -27,7 +28,6 @@ #include "ttime.h" #include "tsdb.h" -#include "tsdbDBDef.h" #include "tsdbCommit.h" #include "tsdbFS.h" #include "tsdbFile.h" @@ -46,7 +46,7 @@ extern "C" { struct STsdb { int32_t vgId; bool repoLocked; - pthread_mutex_t mutex; + TdThreadMutex mutex; char * path; STsdbCfg config; STsdbMemTable * mem; @@ -63,6 +63,8 @@ struct STsdb { #define REPO_ID(r) ((r)->vgId) #define REPO_CFG(r) (&(r)->config) #define REPO_FS(r) (r)->fs +#define REPO_META(r) (r)->pMeta +#define REPO_TFS(r) (r)->pTfs #define IS_REPO_LOCKED(r) (r)->repoLocked #define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv) diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index 96c5f8468f2ac97d047683220acbae6cf2e8dc8a..8156cbae00b19115f9bbad5bce8f2393e1daadcc 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -42,29 +42,29 @@ typedef struct { typedef struct { STsdbFSMeta meta; // FS meta SArray * df; // data file array - SArray * sf; // sma data file array v2(t|r)1900.index_name_1 + SArray * sf; // sma data file array v2f1900.index_name_1 } SFSStatus; /** * @brief Directory structure of .tsma data files. * - * /vnode2/tsdb $ tree .sma/ - * .sma/ - * ├── v2t100.index_name_1 - * ├── v2t101.index_name_1 - * ├── v2t102.index_name_1 - * ├── v2t1900.index_name_3 - * ├── v2t1901.index_name_3 - * ├── v2t1902.index_name_3 - * ├── v2t200.index_name_2 - * ├── v2t201.index_name_2 - * └── v2t202.index_name_2 + * /vnode2/tsdb $ tree tsma/ + * tsma/ + * ├── v2f100.index_name_1 + * ├── v2f101.index_name_1 + * ├── v2f102.index_name_1 + * ├── v2f1900.index_name_3 + * ├── v2f1901.index_name_3 + * ├── v2f1902.index_name_3 + * ├── v2f200.index_name_2 + * ├── v2f201.index_name_2 + * └── v2f202.index_name_2 * * 0 directories, 9 files */ typedef struct { - pthread_rwlock_t lock; + TdThreadRwlock lock; SFSStatus *cstatus; // current status SHashObj * metaCache; // meta cache @@ -108,7 +108,7 @@ SDFileSet *tsdbFSIterNext(SFSIter *pIter); int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_rdlock(&(pFs->lock)); + int code = taosThreadRwlockRdlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -117,7 +117,7 @@ static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { } static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_wrlock(&(pFs->lock)); + int code = taosThreadRwlockWrlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -126,7 +126,7 @@ static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { } static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_unlock(&(pFs->lock)); + int code = taosThreadRwlockUnlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; diff --git a/source/dnode/vnode/src/inc/tsdbLog.h b/source/dnode/vnode/src/inc/tsdbLog.h index 56dc8ab2a0671e35ab64f5119a17658388f3db37..3afe62819884c44d69be6a66e3fd51384c829960 100644 --- a/source/dnode/vnode/src/inc/tsdbLog.h +++ b/source/dnode/vnode/src/inc/tsdbLog.h @@ -24,12 +24,12 @@ extern "C" { extern int32_t tsdbDebugFlag; -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tsdbMemory.h b/source/dnode/vnode/src/inc/tsdbMemory.h index df4df0053f97ef984c209cba3bfb5175a4b5d432..69976fc0783117777b5e89a0c33405d6cfb4f0ae 100644 --- a/source/dnode/vnode/src/inc/tsdbMemory.h +++ b/source/dnode/vnode/src/inc/tsdbMemory.h @@ -30,7 +30,7 @@ static void taosTMemset(void *ptr, int c); static FORCE_INLINE void *taosTMalloc(size_t size) { if (size <= 0) return NULL; - void *ret = malloc(size + sizeof(size_t)); + void *ret = taosMemoryMalloc(size + sizeof(size_t)); if (ret == NULL) return NULL; *(size_t *)ret = size; @@ -58,7 +58,7 @@ static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) { void * tptr = (void *)((char *)ptr - sizeof(size_t)); size_t tsize = size + sizeof(size_t); - void* tptr1 = realloc(tptr, tsize); + void* tptr1 = taosMemoryRealloc(tptr, tsize); if (tptr1 == NULL) return NULL; tptr = tptr1; @@ -69,7 +69,7 @@ static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) { static FORCE_INLINE void* taosTZfree(void* ptr) { if (ptr) { - free((void*)((char*)ptr - sizeof(size_t))); + taosMemoryFree((void*)((char*)ptr - sizeof(size_t))); } return NULL; } diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 649b5a2d47a102ee7fdf34b649609ffc7a08d4c2..e0170c90e72441a3a634f753ff77c15ca53c8cdc 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -16,17 +16,21 @@ #ifndef _TD_TSDB_SMA_H_ #define _TD_TSDB_SMA_H_ -typedef struct SSmaStat SSmaStat; -typedef struct SSmaEnv SSmaEnv; +#define TSDB_SMA_TEST // remove after test finished + +typedef struct SSmaStat SSmaStat; +typedef struct SSmaEnv SSmaEnv; struct SSmaEnv { - pthread_rwlock_t lock; - TDBEnv dbEnv; - char * path; - SSmaStat * pStat; + TdThreadRwlock lock; + SDiskID did; + TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? + char *path; // relative path + SSmaStat *pStat; }; #define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_DID(env) ((env)->did) #define SMA_ENV_ENV(env) ((env)->dbEnv) #define SMA_ENV_PATH(env) ((env)->path) #define SMA_ENV_STAT(env) ((env)->pStat) @@ -49,7 +53,7 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId, } static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_rdlock(&(pEnv->lock)); + int code = taosThreadRwlockRdlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -58,7 +62,7 @@ static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { } static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_wrlock(&(pEnv->lock)); + int code = taosThreadRwlockWrlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -67,7 +71,7 @@ static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { } static FORCE_INLINE int tsdbUnLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_unlock(&(pEnv->lock)); + int code = taosThreadRwlockUnlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 6f4f0049e3b728103122cbf71d9b16f37bd88c53..8d256995c68fd196f8c5df96823816bd48b78cb3 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -23,7 +23,6 @@ #include "tlist.h" #include "tlockfree.h" #include "tmacro.h" -#include "tq.h" #include "wal.h" #include "vnode.h" @@ -34,6 +33,8 @@ extern "C" { #endif +typedef struct STQ STQ; + typedef struct SVState SVState; typedef struct SVBufPool SVBufPool; @@ -46,18 +47,29 @@ typedef struct SVnodeTask { typedef struct SVnodeMgr { td_mode_flag_t vnodeInitFlag; // For commit - bool stop; - uint16_t nthreads; - pthread_t* threads; - pthread_mutex_t mutex; - pthread_cond_t hasTask; + bool stop; + uint16_t nthreads; + TdThread* threads; + TdThreadMutex mutex; + TdThreadCond hasTask; TD_DLIST(SVnodeTask) queue; - // For vnode Mgmt - SDnode* pDnode; - PutReqToVQueryQFp putReqToVQueryQFp; - SendReqToDnodeFp sendReqToDnodeFp; } SVnodeMgr; +typedef struct { + int8_t streamType; // sma or other + int8_t dstType; + int16_t padding; + int32_t smaId; + int64_t tbUid; + int64_t lastReceivedVer; + int64_t lastCommittedVer; +} SStreamSinkInfo; + +typedef struct { + SVnode* pVnode; + SHashObj* pHash; // streamId -> SStreamSinkInfo +} SSink; + extern SVnodeMgr vnodeMgr; // SVState @@ -75,19 +87,17 @@ struct SVnode { SVBufPool* pBufPool; SMeta* pMeta; STsdb* pTsdb; - STQ* pTq; SWal* pWal; + STQ* pTq; + SSink* pSink; tsem_t canCommit; SQHandle* pQuery; - SDnode* pDnode; + SMsgCb msgCb; STfs* pTfs; }; int vnodeScheduleTask(SVnodeTask* task); -int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); -void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); - #define vFatal(...) \ do { \ if (vDebugFlag & DEBUG_FATAL) { \ @@ -172,6 +182,26 @@ void* vmaMalloc(SVMemAllocator* pVMA, uint64_t size); void vmaFree(SVMemAllocator* pVMA, void* ptr); bool vmaIsFull(SVMemAllocator* pVMA); +// init once +int tqInit(); +void tqCleanUp(); + +// open in each vnode +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, + SMemAllocatorFactory* allocFac); +void tqClose(STQ*); + +// required by vnode +int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); +int tqCommit(STQ*); + +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessSetConnReq(STQ* pTq, char* msg); +int32_t tqProcessRebReq(STQ* pTq, char* msg); +int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg); +int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 99a2b272ed36313dfb3c940d08e35a834b3914bc..fdcc1652072f127d99863957b609944845572f31 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -33,7 +33,7 @@ typedef struct { struct SMetaDB { #if IMPL_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif // DB DB *pTbDB; @@ -233,7 +233,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { // save sma info int32_t len = tEncodeTSma(NULL, pSmaCfg); - pBuf = calloc(len, 1); + pBuf = taosMemoryCalloc(len, 1); if (pBuf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -254,12 +254,12 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { metaDBULock(pMeta->pDB); // release - tfree(pBuf); + taosMemoryFreeClear(pBuf); return 0; } -int metaRemoveSmaFromDb(SMeta *pMeta, const char *indexName) { +int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) { // TODO #if 0 DBT key = {0}; @@ -297,7 +297,7 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { SSchema *pSchema; buf = taosDecodeFixedU32(buf, &pSW->nCols); - pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * pSW->nCols); + pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; buf = taosDecodeFixedI8(buf, &pSchema->type); @@ -311,13 +311,13 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { static SMetaDB *metaNewDB() { SMetaDB *pDB = NULL; - pDB = (SMetaDB *)calloc(1, sizeof(*pDB)); + pDB = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDB)); if (pDB == NULL) { return NULL; } #if IMPL_WITH_LOCK - pthread_rwlock_init(&pDB->rwlock, NULL); + taosThreadRwlockInit(&pDB->rwlock, NULL); #endif return pDB; @@ -326,9 +326,9 @@ static SMetaDB *metaNewDB() { static void metaFreeDB(SMetaDB *pDB) { if (pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_destroy(&pDB->rwlock); + taosThreadRwlockDestroy(&pDB->rwlock); #endif - free(pDB); + taosMemoryFree(pDB); } } @@ -463,7 +463,7 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey DBT *pDbt; if (pTbCfg->type == META_CHILD_TABLE) { - // pDbt = calloc(2, sizeof(DBT)); + // pDbt = taosMemoryCalloc(2, sizeof(DBT)); // // First key is suid // pDbt[0].data = &(pTbCfg->ctbCfg.suid); @@ -507,7 +507,7 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { tsize += taosEncodeString(buf, pTbCfg->name); tsize += taosEncodeFixedU32(buf, pTbCfg->ttl); tsize += taosEncodeFixedU32(buf, pTbCfg->keep); - tsize += taosEncodeFixedU8(buf, pTbCfg->type); + tsize += taosEncodeFixedU8(buf, pTbCfg->info); if (pTbCfg->type == META_SUPER_TABLE) { SSchemaWrapper sw = {.nCols = pTbCfg->stbCfg.nTagCols, .pSchema = pTbCfg->stbCfg.pTagSchema}; @@ -527,7 +527,7 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { buf = taosDecodeString(buf, &(pTbCfg->name)); buf = taosDecodeFixedU32(buf, &(pTbCfg->ttl)); buf = taosDecodeFixedU32(buf, &(pTbCfg->keep)); - buf = taosDecodeFixedU8(buf, &(pTbCfg->type)); + buf = taosDecodeFixedU8(buf, &(pTbCfg->info)); if (pTbCfg->type == META_SUPER_TABLE) { SSchemaWrapper sw; @@ -545,11 +545,11 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { } static void metaClearTbCfg(STbCfg *pTbCfg) { - tfree(pTbCfg->name); + taosMemoryFreeClear(pTbCfg->name); if (pTbCfg->type == META_SUPER_TABLE) { tdFreeSchema(pTbCfg->stbCfg.pTagSchema); } else if (pTbCfg->type == META_CHILD_TABLE) { - tfree(pTbCfg->ctbCfg.pTag); + taosMemoryFreeClear(pTbCfg->ctbCfg.pTag); } } @@ -574,7 +574,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { } // Decode - pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + pTbCfg = (STbCfg *)taosMemoryMalloc(sizeof(*pTbCfg)); if (pTbCfg == NULL) { return NULL; } @@ -606,7 +606,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { // Decode *uid = *(tb_uid_t *)(pkey.data); - pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + pTbCfg = (STbCfg *)taosMemoryMalloc(sizeof(*pTbCfg)); if (pTbCfg == NULL) { return NULL; } @@ -636,13 +636,13 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { } // Decode - pCfg = (STSma *)calloc(1, sizeof(STSma)); + pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); if (pCfg == NULL) { return NULL; } if (tDecodeTSma(value.data, pCfg) == NULL) { - tfree(pCfg); + taosMemoryFreeClear(pCfg); return NULL; } @@ -675,7 +675,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo // Decode the schema pBuf = value.data; - pSW = malloc(sizeof(*pSW)); + pSW = taosMemoryMalloc(sizeof(*pSW)); metaDecodeSchema(pBuf, pSW); return pSW; @@ -689,7 +689,7 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { SMTbCursor *pTbCur = NULL; SMetaDB *pDB = pMeta->pDB; - pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); + pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); if (pTbCur == NULL) { return NULL; } @@ -705,12 +705,24 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { return pTbCur; } +int metaGetTbNum(SMeta *pMeta) { + SMetaDB *pDB = pMeta->pDB; + + DB_BTREE_STAT *sp1; + pDB->pTbDB->stat(pDB->pNtbIdx, NULL, &sp1, 0); + + DB_BTREE_STAT *sp2; + pDB->pTbDB->stat(pDB->pCtbIdx, NULL, &sp2, 0); + + return sp1->bt_nkeys + sp2->bt_nkeys; +} + void metaCloseTbCursor(SMTbCursor *pTbCur) { if (pTbCur) { if (pTbCur->pCur) { pTbCur->pCur->close(pTbCur->pCur); } - free(pTbCur); + taosMemoryFree(pTbCur); } } @@ -725,8 +737,8 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { - free(tbCfg.name); - free(tbCfg.stbCfg.pTagSchema); + taosMemoryFree(tbCfg.name); + taosMemoryFree(tbCfg.stbCfg.pTagSchema); continue; } else if (tbCfg.type == META_CHILD_TABLE) { kvRowFree(tbCfg.ctbCfg.pTag); @@ -780,7 +792,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMetaDB *pDB = pMeta->pDB; int ret; - pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); + pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); if (pCtbCur == NULL) { return NULL; } @@ -788,7 +800,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { pCtbCur->suid = uid; ret = pDB->pCtbIdx->cursor(pDB->pCtbIdx, NULL, &(pCtbCur->pCur), 0); if (ret != 0) { - free(pCtbCur); + taosMemoryFree(pCtbCur); return NULL; } @@ -801,7 +813,7 @@ void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { pCtbCur->pCur->close(pCtbCur->pCur); } - free(pCtbCur); + taosMemoryFree(pCtbCur); } } @@ -837,7 +849,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { SMetaDB *pDB = pMeta->pDB; int ret; - pCur = (SMSmaCursor *)calloc(1, sizeof(*pCur)); + pCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pCur)); if (pCur == NULL) { return NULL; } @@ -846,7 +858,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { // TODO: lock? ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0); if (ret != 0) { - free(pCur); + taosMemoryFree(pCur); return NULL; } @@ -859,7 +871,7 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) { pCur->pCur->close(pCur->pCur); } - free(pCur); + taosMemoryFree(pCur); } } @@ -884,14 +896,14 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) { STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { STSmaWrapper *pSW = NULL; - pSW = calloc(1, sizeof(*pSW)); + pSW = taosMemoryCalloc(1, sizeof(*pSW)); if (pSW == NULL) { return NULL; } SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); if (pCur == NULL) { - free(pSW); + taosMemoryFree(pSW); return NULL; } @@ -903,11 +915,11 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { // TODO: lock? if (pCur->pCur->pget(pCur->pCur, &skey, NULL, &pval, DB_NEXT) == 0) { ++pSW->number; - STSma *tptr = (STSma *)realloc(pSW->tSma, pSW->number * sizeof(STSma)); + STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { metaCloseSmaCurosr(pCur); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); return NULL; } pSW->tSma = tptr; @@ -915,7 +927,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { metaCloseSmaCurosr(pCur); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); return NULL; } continue; @@ -965,18 +977,18 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { static void metaDBWLock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_wrlock(&(pDB->rwlock)); + taosThreadRwlockWrlock(&(pDB->rwlock)); #endif } static void metaDBRLock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_rdlock(&(pDB->rwlock)); + taosThreadRwlockRdlock(&(pDB->rwlock)); #endif } static void metaDBULock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_unlock(&(pDB->rwlock)); + taosThreadRwlockUnlock(&(pDB->rwlock)); #endif } diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 881ea4f46dcac373a7fe3fad487ae487da2ab515..818da147381c46f1aab1816407861565f071bd81 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -121,11 +121,11 @@ int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) { return TSDB_CODE_SUCCESS; } -int32_t metaDropTSma(SMeta *pMeta, char* indexName) { +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { // TODO: Validate the cfg // TODO: add atomicity - if (metaRemoveSmaFromDb(pMeta, indexName) < 0) { + if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) { // TODO: handle error return -1; } diff --git a/source/dnode/vnode/src/meta/metaMain.c b/source/dnode/vnode/src/meta/metaMain.c index ad87b2de9ec0861293290a410b004395a882a6e3..690b96bbb058a7a74b122a73d9b604da8c18d8bb 100644 --- a/source/dnode/vnode/src/meta/metaMain.c +++ b/source/dnode/vnode/src/meta/metaMain.c @@ -69,7 +69,7 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorF SMeta *pMeta; size_t psize = strlen(path); - pMeta = (SMeta *)calloc(1, sizeof(*pMeta)); + pMeta = (SMeta *)taosMemoryCalloc(1, sizeof(*pMeta)); if (pMeta == NULL) { return NULL; } @@ -88,8 +88,8 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorF static void metaFree(SMeta *pMeta) { if (pMeta) { - tfree(pMeta->path); - free(pMeta); + taosMemoryFreeClear(pMeta->path); + taosMemoryFree(pMeta); } } diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 4a65cf277b9dd11aeb9a0ee47cea6e19f314d10c..f4b450b4a8ee326c965d78e52dcab41940b9bd5a 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -47,7 +47,7 @@ int metaOpenDB(SMeta *pMeta) { TDB * pCtbIdx; int ret; - pDb = (SMetaDB *)calloc(1, sizeof(*pDb)); + pDb = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8cdc250e8d5bbb9cbaac311a38ac2001144572a1..91cbc2cff8018b20f7b33c9f21179a61bea2c1a3 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -14,21 +14,25 @@ */ #include "tcompare.h" +#include "tdatablock.h" #include "tqInt.h" #include "tqMetaStore.h" +#include "tstream.h" int32_t tqInit() { return tqPushMgrInit(); } void tqCleanUp() { tqPushMgrCleanUp(); } -STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { - STQ* pTq = malloc(sizeof(STQ)); +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, + SMemAllocatorFactory* allocFac) { + STQ* pTq = taosMemoryMalloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } pTq->path = strdup(path); pTq->tqConfig = tqConfig; + pTq->pVnode = pVnode; pTq->pWal = pWal; pTq->pVnodeMeta = pVnodeMeta; #if 0 @@ -39,21 +43,23 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S } #endif pTq->tqMeta = - tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0); + tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); if (pTq->tqMeta == NULL) { - free(pTq); + taosMemoryFree(pTq); #if 0 allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); #endif return NULL; } +#if 0 pTq->tqPushMgr = tqPushMgrOpen(); if (pTq->tqPushMgr == NULL) { // free store - free(pTq); + taosMemoryFree(pTq); return NULL; } +#endif pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -62,21 +68,34 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S void tqClose(STQ* pTq) { if (pTq) { - tfree(pTq->path); - free(pTq); + taosMemoryFreeClear(pTq->path); + taosMemoryFree(pTq); } // TODO } -int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { +int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) { if (msgType != TDMT_VND_SUBMIT) return 0; + void* data = taosMemoryMalloc(msgLen); + if (data == NULL) { + return -1; + } + memcpy(data, msg, msgLen); + SRpcMsg req = { + .msgType = TDMT_VND_STREAM_TRIGGER, + .pCont = data, + .contLen = msgLen, + }; + tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req); + +#if 0 void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL); while (pIter != NULL) { STqPusher* pusher = *(STqPusher**)pIter; if (pusher->type == TQ_PUSHER_TYPE__STREAM) { STqStreamPusher* streamPusher = (STqStreamPusher*)pusher; // repack - STqStreamToken* token = malloc(sizeof(STqStreamToken)); + STqStreamToken* token = taosMemoryMalloc(sizeof(STqStreamToken)); if (token == NULL) { taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -97,6 +116,7 @@ int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { // if handle waiting, launch query and response to consumer // // if no waiting handle, return +#endif return 0; } @@ -179,9 +199,9 @@ int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead int32_t sz = tEncodeSTqConsumer(NULL, pConsumer); if (sz > (*ppHead)->ssize) { - void* tmpPtr = realloc(*ppHead, sizeof(STqSerializedHead) + sz); + void* tmpPtr = taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sz); if (tmpPtr == NULL) { - free(*ppHead); + taosMemoryFree(*ppHead); terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } @@ -198,7 +218,7 @@ int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) { const void* str = pHead->content; - *ppConsumer = calloc(1, sizeof(STqConsumer)); + *ppConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); if (*ppConsumer == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -245,7 +265,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } SMqPollRsp rsp = { - .consumerId = consumerId, + /*.consumerId = consumerId,*/ .numOfTopics = 0, .pBlockData = NULL, }; @@ -282,7 +302,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (pHead->head.msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; qTaskInfo_t task = pTopic->buffer.output[pos].task; - qSetStreamInput(task, pCont); + qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); while (1) { SSDataBlock* pDataBlock; @@ -298,7 +318,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } taosArrayPush(pRes, pDataBlock); - rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; + rsp.schema = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; rsp.rspOffset = fetchOffset; rsp.numOfTopics = 1; @@ -312,10 +332,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = consumerId; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqPollRsp(&abuf, &rsp); - taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); + /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; @@ -371,7 +392,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { tDecodeSMqSetCVgReq(msg, &req); /*printf("vg %d set to consumer from %ld to %ld\n", req.vgId, req.oldConsumerId, req.newConsumerId);*/ - STqConsumer* pConsumer = calloc(1, sizeof(STqConsumer)); + STqConsumer* pConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); if (pConsumer == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -382,10 +403,10 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pConsumer->consumerId = req.consumerId; pConsumer->epoch = 0; - STqTopic* pTopic = calloc(1, sizeof(STqTopic)); + STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic)); if (pTopic == NULL) { taosArrayDestroy(pConsumer->topics); - free(pConsumer); + taosMemoryFree(pConsumer); return -1; } strcpy(pTopic->topicName, req.topicName); @@ -419,17 +440,71 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { return 0; } +int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { + if (pTask->execType == TASK_EXEC__NONE) return 0; + + pTask->exec.numOfRunners = parallel; + pTask->exec.runners = taosMemoryCalloc(parallel, sizeof(SStreamRunner)); + if (pTask->exec.runners == NULL) { + return -1; + } + for (int32_t i = 0; i < parallel; i++) { + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); + SReadHandle handle = { + .reader = pReadHandle, + .meta = pTq->pVnodeMeta, + }; + pTask->exec.runners[i].inputHandle = pReadHandle; + pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); + ASSERT(pTask->exec.runners[i].executor); + } + return 0; +} + int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { - SStreamTask* pTask = malloc(sizeof(SStreamTask)); + SStreamTask* pTask = taosMemoryMalloc(sizeof(SStreamTask)); if (pTask == NULL) { return -1; } SCoder decoder; tCoderInit(&decoder, TD_LITTLE_ENDIAN, (uint8_t*)msg, msgLen, TD_DECODER); - tDecodeSStreamTask(&decoder, pTask); + if (tDecodeSStreamTask(&decoder, pTask) < 0) { + ASSERT(0); + } tCoderClear(&decoder); + if (tqExpandTask(pTq, pTask, 4) < 0) { + ASSERT(0); + } + taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask)); return 0; } + +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen) { + void* pIter = NULL; + + while (1) { + pIter = taosHashIterate(pTq->pStreamTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = (SStreamTask*)pIter; + + if (streamExecTask(pTask, &pTq->pVnode->msgCb, data, STREAM_DATA_TYPE_SUBMIT_BLOCK, 0) < 0) { + // TODO + } + } + return 0; +} + +int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg) { + SStreamTaskExecReq* pReq = msg->pCont; + int32_t taskId = pReq->taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + ASSERT(pTask); + + if (streamExecTask(pTask, &pTq->pVnode->msgCb, pReq->data, STREAM_DATA_TYPE_SSDATA_BLOCK, 0) < 0) { + // TODO + } + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index ce00c98ff92b8293cfdb52fd47ad6de479e539e3..505687755dd9ff4499590d8fee9388fb8dd09cb9 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -70,7 +70,7 @@ static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter, int32_t tqConfigFlag) { - STqMetaStore* pMeta = calloc(1, sizeof(STqMetaStore)); + STqMetaStore* pMeta = taosMemoryCalloc(1, sizeof(STqMetaStore)); if (pMeta == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; @@ -79,10 +79,10 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F // concat data file name and index file name size_t pathLen = strlen(path); - pMeta->dirPath = malloc(pathLen + 1); + pMeta->dirPath = taosMemoryMalloc(pathLen + 1); if (pMeta->dirPath == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - free(pMeta); + taosMemoryFree(pMeta); return NULL; } strcpy(pMeta->dirPath, path); @@ -104,7 +104,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } pMeta->pIdxFile = pIdxFile; - pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); + pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; @@ -129,7 +129,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F // read idx file and load into memory STqIdxPageBuf idxBuf; - STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); + STqSerializedHead* serializedObj = taosMemoryMalloc(TQ_PAGE_SIZE); if (serializedObj == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; } @@ -145,7 +145,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F ASSERT(idxBuf.head.writeOffset == idxRead); // loop read every entry for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList* pNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // TODO: free memory @@ -154,7 +154,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); if (allocated < pNode->handle.serializedSize) { - void* ptr = realloc(serializedObj, pNode->handle.serializedSize); + void* ptr = taosMemoryRealloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // TODO: free memory @@ -225,11 +225,11 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pBucketNode->handle.valueInTxn); } - free(pBucketNode); + taosMemoryFree(pBucketNode); } } } - free(serializedObj); + taosMemoryFree(serializedObj); return pMeta; } @@ -252,13 +252,13 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; - free(pNode); + taosMemoryFree(pNode); pNode = next; } } - free(pMeta->dirPath); - free(pMeta->unpersistHead); - free(pMeta); + taosMemoryFree(pMeta->dirPath); + taosMemoryFree(pMeta->unpersistHead); + taosMemoryFree(pMeta); return 0; } @@ -277,14 +277,14 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; - free(pNode); + taosMemoryFree(pNode); pNode = next; } } - free(pMeta->unpersistHead); + taosMemoryFree(pMeta->unpersistHead); taosRemoveDir(pMeta->dirPath); - free(pMeta->dirPath); - free(pMeta); + taosMemoryFree(pMeta->dirPath); + taosMemoryFree(pMeta); return 0; } @@ -293,7 +293,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { int64_t* bufPtr = (int64_t*)idxBuf.buffer; STqMetaList* pHead = pMeta->unpersistHead; STqMetaList* pNode = pHead->unpersistNext; - STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); + STqSerializedHead* pSHead = taosMemoryMalloc(sizeof(STqSerializedHead)); if (pSHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -383,12 +383,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { ASSERT(pBucketNode->next == pNode); pBucketNode->next = pNode->next; } - free(pNode); + taosMemoryFree(pNode); } } // write left bytes - free(pSHead); + taosMemoryFree(pSHead); // TODO: write new version in tfile if ((char*)bufPtr != idxBuf.buffer) { int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); @@ -416,7 +416,7 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu pNode = pNode->next; } } - STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -488,7 +488,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va pNode = pNode->next; } } - STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -504,7 +504,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); } int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { - void* vmem = malloc(vsize); + void* vmem = taosMemoryMalloc(vsize); if (vmem == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 4115cb73137b8c02269f372e0da5abfaf154a4b1..20270950cdc6ff13208f7d86bcfee48c28dac55e 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -31,7 +31,7 @@ struct STqOffsetStore { }; STqOffsetStore* STqOffsetOpen(STqOffsetCfg* pCfg) { - STqOffsetStore* pStore = malloc(sizeof(STqOffsetStore)); + STqOffsetStore* pStore = taosMemoryMalloc(sizeof(STqOffsetStore)); if (pStore == NULL) { return NULL; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 4186f29e2aa7111ce53f143247bbaf5f15018d66..7851c071c340aa2fb0bc65cf9cace22166f5bd0d 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -32,7 +32,7 @@ void tqPushMgrCleanUp() { } STqPushMgr* tqPushMgrOpen() { - STqPushMgr* mgr = malloc(sizeof(STqPushMgr)); + STqPushMgr* mgr = taosMemoryMalloc(sizeof(STqPushMgr)); if (mgr == NULL) { return NULL; } @@ -42,11 +42,11 @@ STqPushMgr* tqPushMgrOpen() { void tqPushMgrClose(STqPushMgr* pushMgr) { taosHashCleanup(pushMgr->pHash); - free(pushMgr); + taosMemoryFree(pushMgr); } STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) { - STqClientPusher* clientPusher = malloc(sizeof(STqClientPusher)); + STqClientPusher* clientPusher = taosMemoryMalloc(sizeof(STqClientPusher)); if (clientPusher == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -57,7 +57,7 @@ STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t c clientPusher->ttl = ttl; if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(clientPusher); + taosMemoryFree(clientPusher); // TODO send rsp back return NULL; } @@ -65,7 +65,7 @@ STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t c } STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) { - STqStreamPusher* streamPusher = malloc(sizeof(STqStreamPusher)); + STqStreamPusher* streamPusher = taosMemoryMalloc(sizeof(STqStreamPusher)); if (streamPusher == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -77,7 +77,7 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(streamPusher); + taosMemoryFree(streamPusher); return NULL; } return streamPusher; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0c4b933c198bf47aa9e8590afb186d66b28c2fc8..37e7ed11ae6ed04c532bc8ed771ac9bded12942d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -17,7 +17,7 @@ #include "vnode.h" STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { - STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); + STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { return NULL; } @@ -44,7 +44,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t if (pReadHandle->pBlock == NULL) break; pReadHandle->pBlock->uid = htobe64(pReadHandle->pBlock->uid); - pReadHandle->pBlock->tid = htonl(pReadHandle->pBlock->tid); + pReadHandle->pBlock->suid = htobe64(pReadHandle->pBlock->suid); pReadHandle->pBlock->sversion = htonl(pReadHandle->pBlock->sversion); pReadHandle->pBlock->dataLen = htonl(pReadHandle->pBlock->dataLen); pReadHandle->pBlock->schemaLen = htonl(pReadHandle->pBlock->schemaLen); @@ -126,6 +126,36 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { if (pArray == NULL) { return NULL; } + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + int16_t colIdSchema = pColSchema->colId; + int16_t colIdNeed = *(int16_t*)taosArrayGet(pHandle->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = {0}; + int sz = numOfRows * pColSchema->bytes; + colInfo.info.bytes = pColSchema->bytes; + colInfo.info.colId = pColSchema->colId; + colInfo.info.type = pColSchema->type; + + colInfo.pData = taosMemoryCalloc(1, sz); + if (colInfo.pData == NULL) { + // TODO free + taosArrayDestroy(pArray); + return NULL; + } + + blockDataEnsureColumnCapacity(&colInfo, numOfRows); + taosArrayPush(pArray, &colInfo); + colMeta++; + colNeed++; + } + } int j = 0; for (int32_t i = 0; i < colNumNeed; i++) { @@ -143,7 +173,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colInfo.info.colId = colId; colInfo.info.type = pColSchema->type; - colInfo.pData = calloc(1, sz); + colInfo.pData = taosMemoryCalloc(1, sz); if (colInfo.pData == NULL) { // TODO free taosArrayDestroy(pArray); @@ -163,11 +193,23 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block + int32_t colTot = taosArrayGetSize(pArray); + for (int32_t i = 0; i < colTot; i++) { + SColumnInfoData* pColData = taosArrayGet(pArray, i); + SCellVal sVal = {0}; + if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { + break; + } + memcpy(POINTER_SHIFT(pColData->pData, curRow * pColData->info.bytes), sVal.val, pColData->info.bytes); + } +#if 0 for (int32_t i = 0; i < colNumNeed; i++) { SColumnInfoData* pColData = taosArrayGet(pArray, i); STColumn* pCol = schemaColAt(pTschema, i); // TODO - ASSERT(pCol->colId == pColData->info.colId); + if(pCol->colId != pColData->info.colId) { + continue; + } // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) { @@ -176,6 +218,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { } memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes); } +#endif curRow++; } return pArray; diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index ee279abf47ff6acfe406d9adff26bdf4dd9ed59b..c8f4cd642ac5bea567bf1a470df0a8df42e91095 100644 --- a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -51,7 +51,7 @@ void tsdbCloseDBF(SDBFile *pDBF) { tsdbCloseBDBDb(pDBF->pDB); pDBF->pDB = NULL; } - tfree(pDBF->path); + taosMemoryFreeClear(pDBF->path); } int32_t tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { @@ -159,7 +159,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, void* key, uint32_t keySize, uint32_t * return NULL; } - result = calloc(1, value1.size); + result = taosMemoryCalloc(1, value1.size); if (result == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 37403f1a11955cc418e92a371da15bdc57fbf624..eca0a9612f4398f9db57732450decea1975ba3d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -403,7 +403,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { STbData *pTbData; pCommith->niters = SL_SIZE(pMem->pSlIdx); - pCommith->iters = (SCommitIter *)calloc(pCommith->niters, sizeof(SCommitIter)); + pCommith->iters = (SCommitIter *)taosMemoryCalloc(pCommith->niters, sizeof(SCommitIter)); if (pCommith->iters == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -424,7 +424,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); tSkipListIterNext(pCommitIter->pIter); - pCommitIter->pTable = (STable *)malloc(sizeof(STable)); + pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable)); pCommitIter->pTable->uid = pTbData->uid; pCommitIter->pTable->tid = pTbData->uid; pCommitIter->pTable->pSchema = metaGetTbTSchema(pRepo->pMeta, pTbData->uid, 0); @@ -439,10 +439,10 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) { for (int i = 1; i < pCommith->niters; i++) { tSkipListDestroyIter(pCommith->iters[i].pIter); tdFreeSchema(pCommith->iters[i].pTable->pSchema); - free(pCommith->iters[i].pTable); + taosMemoryFree(pCommith->iters[i].pTable); } - free(pCommith->iters); + taosMemoryFree(pCommith->iters); pCommith->iters = NULL; pCommith->niters = 0; } @@ -985,7 +985,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // SKVRecord *pRecord; // void *pBuf = NULL; -// pBuf = malloc((size_t)maxBufSize); +// pBuf = taosMemoryMalloc((size_t)maxBufSize); // if (pBuf == NULL) { // goto _err; // } @@ -1006,7 +1006,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // } // if (pRecord->size > maxBufSize) { // maxBufSize = pRecord->size; -// void* tmp = realloc(pBuf, (size_t)maxBufSize); +// void* tmp = taosMemoryRealloc(pBuf, (size_t)maxBufSize); // if (tmp == NULL) { // goto _err; // } @@ -1059,7 +1059,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // pfs->metaCacheComp = NULL; // } -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // ASSERT(mf.info.nDels == 0); // ASSERT(mf.info.tombSize == 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index a03739c90feb9155040e28361f82cd3ce1c80e23..aa235f88de98e950b9bf8730538d2017325b2c6d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -14,8 +14,8 @@ */ #include -#include "os.h" #include "tsdbDef.h" +#include "os.h" typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; @@ -134,7 +134,7 @@ static void *tsdbDecodeFSStatus(STsdb*pRepo, void *buf, SFSStatus *pStatus) { } static SFSStatus *tsdbNewFSStatus(int maxFSet) { - SFSStatus *pStatus = (SFSStatus *)calloc(1, sizeof(*pStatus)); + SFSStatus *pStatus = (SFSStatus *)taosMemoryCalloc(1, sizeof(*pStatus)); if (pStatus == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -145,7 +145,7 @@ static SFSStatus *tsdbNewFSStatus(int maxFSet) { pStatus->df = taosArrayInit(maxFSet, sizeof(SDFileSet)); if (pStatus->df == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - free(pStatus); + taosMemoryFree(pStatus); return NULL; } @@ -155,7 +155,7 @@ static SFSStatus *tsdbNewFSStatus(int maxFSet) { static SFSStatus *tsdbFreeFSStatus(SFSStatus *pStatus) { if (pStatus) { pStatus->df = taosArrayDestroy(pStatus->df); - free(pStatus); + taosMemoryFree(pStatus); } return NULL; @@ -197,16 +197,16 @@ STsdbFS *tsdbNewFS(const STsdbCfg *pCfg) { int maxFSet = TSDB_MAX_FSETS(keep, days); STsdbFS *pfs; - pfs = (STsdbFS *)calloc(1, sizeof(*pfs)); + pfs = (STsdbFS *)taosMemoryCalloc(1, sizeof(*pfs)); if (pfs == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } - int code = pthread_rwlock_init(&(pfs->lock), NULL); + int code = taosThreadRwlockInit(&(pfs->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); - free(pfs); + taosMemoryFree(pfs); return NULL; } @@ -241,8 +241,8 @@ void *tsdbFreeFS(STsdbFS *pfs) { taosHashCleanup(pfs->metaCache); pfs->metaCache = NULL; pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); - pthread_rwlock_destroy(&(pfs->lock)); - free(pfs); + taosThreadRwlockDestroy(&(pfs->lock)); + taosMemoryFree(pfs); } return NULL; @@ -853,7 +853,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // } // if (recoverMeta) { -// pBuf = malloc((size_t)maxBufSize); +// pBuf = taosMemoryMalloc((size_t)maxBufSize); // if (pBuf == NULL) { // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; // tsdbCloseMFile(pMFile); @@ -865,7 +865,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { // tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -874,7 +874,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (nread < 0) { // tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -883,7 +883,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", // REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); // terrno = TSDB_CODE_TDB_FILE_CORRUPTED; -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -891,7 +891,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) { // tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid, // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -903,7 +903,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // } // tsdbCloseMFile(pMFile); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // return 0; // } @@ -1289,7 +1289,7 @@ static int tsdbRestoreCurrent(STsdb *pRepo) { } if (tsdbSaveFSStatus(pRepo, pRepo->fs->cstatus) < 0) { - tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 00e97c7b614b9277040a38f2487ea620d1c2777f..271616e9c2a031cd46dad5e849771510b320d30c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -85,7 +85,7 @@ void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) { strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN); TSDB_FILE_SET_CLOSED(pMFile); - tfree(aname); + taosMemoryFreeClear(aname); return buf; } @@ -119,10 +119,10 @@ int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) { // Try to create directory recursively char *s = strdup(TFILE_REL_NAME(&(pMFile->f))); if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) { - tfree(s); + taosMemoryFreeClear(s); return -1; } - tfree(s); + taosMemoryFreeClear(s); pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); if (pMFile->fd < 0) { @@ -352,7 +352,7 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { buf = taosDecodeString(buf, &aname); strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN); TSDB_FILE_SET_CLOSED(pDFile); - tfree(aname); + taosMemoryFreeClear(aname); return buf; } @@ -366,10 +366,10 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) { - tfree(s); + taosMemoryFreeClear(s); return -1; } - tfree(s); + taosMemoryFreeClear(s); pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pDFile->pFile == NULL) { @@ -692,7 +692,7 @@ int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype } } - tfree(p); + taosMemoryFreeClear(p); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index afa8921c004aa37282d86ff44f2d1f4850f2dfdc..526109f796021cc4e17704717a02ef413ff8806e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -68,7 +68,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMeta *pMeta, STfs *pTfs) { STsdb *pTsdb = NULL; - pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); + pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -93,8 +93,8 @@ static void tsdbFree(STsdb *pTsdb) { tsdbFreeSmaEnv(pTsdb->pRSmaEnv); tsdbFreeSmaEnv(pTsdb->pTSmaEnv); tsdbFreeFS(pTsdb->fs); - tfree(pTsdb->path); - free(pTsdb); + taosMemoryFreeClear(pTsdb->path); + taosMemoryFree(pTsdb); } } @@ -110,7 +110,7 @@ static void tsdbCloseImpl(STsdb *pTsdb) { } int tsdbLockRepo(STsdb *pTsdb) { - int code = pthread_mutex_lock(&pTsdb->mutex); + int code = taosThreadMutexLock(&pTsdb->mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -123,7 +123,7 @@ int tsdbLockRepo(STsdb *pTsdb) { int tsdbUnlockRepo(STsdb *pTsdb) { ASSERT(IS_REPO_LOCKED(pTsdb)); pTsdb->repoLocked = false; - int code = pthread_mutex_unlock(&pTsdb->mutex); + int code = taosThreadMutexUnlock(&pTsdb->mutex); if (code != 0) { tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -298,7 +298,7 @@ STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { } int tsdbLockRepo(STsdbRepo *pRepo) { - int code = pthread_mutex_lock(&pRepo->mutex); + int code = taosThreadMutexLock(&pRepo->mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -311,7 +311,7 @@ int tsdbLockRepo(STsdbRepo *pRepo) { int tsdbUnlockRepo(STsdbRepo *pRepo) { ASSERT(IS_REPO_LOCKED(pRepo)); pRepo->repoLocked = false; - int code = pthread_mutex_unlock(&pRepo->mutex); + int code = taosThreadMutexUnlock(&pRepo->mutex); if (code != 0) { tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -388,7 +388,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { tsdbError("vgId:%d no config changed", REPO_ID(repo)); } - int code = pthread_mutex_lock(&repo->save_mutex); + int code = taosThreadMutexLock(&repo->save_mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -416,7 +416,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { repo->config_changed = true; - pthread_mutex_unlock(&repo->save_mutex); + taosThreadMutexUnlock(&repo->save_mutex); // schedule a commit msg and wait for the new config applied tsdbSyncCommitConfig(repo); @@ -507,7 +507,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t magic = pFile->info.magic; char *tfname = strdup(fname); sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); + taosMemoryFreeClear(tfname); } else { if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { SFile *pFile = &pFGroup->files[0]; @@ -516,17 +516,17 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t magic = pFile->info.magic; char *tfname = strdup(fname); sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); + taosMemoryFreeClear(tfname); } else { return 0; } } } } else { // get the named file at the specified index. If not there, return 0 - fname = malloc(256); + fname = taosMemoryMalloc(256); sprintf(fname, "%s/vnode/vnode%d/%s", tfsGetPrimaryPath(pRepo->pTfs), REPO_ID(pRepo), name); if (access(fname, F_OK) != 0) { - tfree(fname); + taosMemoryFreeClear(fname); return 0; } if (*index == TSDB_META_FILE_INDEX) { // get meta file @@ -536,19 +536,19 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); tsdbGetFileInfoImpl(tfname, &magic, size); } - tfree(fname); + taosMemoryFreeClear(fname); return magic; } if (stat(fname, &fState) < 0) { - tfree(fname); + taosMemoryFreeClear(fname); return 0; } *size = fState.st_size; // magic = *size; - tfree(fname); + taosMemoryFreeClear(fname); return magic; #endif } @@ -674,7 +674,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); + STsdbRepo *pRepo = (STsdbRepo *)taosMemoryCalloc(1, sizeof(*pRepo)); if (pRepo == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -690,14 +690,14 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { pRepo->repoLocked = false; pRepo->pthread = NULL; - int code = pthread_mutex_init(&(pRepo->mutex), NULL); + int code = taosThreadMutexInit(&(pRepo->mutex), NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); tsdbFreeRepo(pRepo); return NULL; } - code = pthread_mutex_init(&(pRepo->save_mutex), NULL); + code = taosThreadMutexInit(&(pRepo->save_mutex), NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); tsdbFreeRepo(pRepo); @@ -747,8 +747,8 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); tsem_destroy(&(pRepo->readyToCommit)); - pthread_mutex_destroy(&pRepo->mutex); - free(pRepo); + taosThreadMutexDestroy(&pRepo->mutex); + taosMemoryFree(pRepo); } } @@ -820,7 +820,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea goto out; } - pBlockStatis = calloc(numColumns, sizeof(SDataStatis)); + pBlockStatis = taosMemoryCalloc(numColumns, sizeof(SDataStatis)); if (pBlockStatis == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; err = -1; @@ -886,7 +886,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // save not-null column uint16_t bytes = IS_VAR_DATA_TYPE(pCol->type) ? varDataTLen(pColData) : pCol->bytes; SDataCol *pLastCol = &(pTable->lastCols[idx]); - pLastCol->pData = malloc(bytes); + pLastCol->pData = taosMemoryMalloc(bytes); pLastCol->bytes = bytes; pLastCol->colId = pCol->colId; memcpy(pLastCol->pData, value, bytes); @@ -907,7 +907,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea out: taosTZfree(row); - tfree(pBlockStatis); + taosMemoryFreeClear(pBlockStatis); if (err == 0 && numColumns <= pTable->restoreColumnNum) { pTable->hasRestoreLastColumn = true; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 01813af556715dd2e86b8203cd314f0428ffb327..6eb721fcb1eb35d84b410821d8f5413b402eec6a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -25,7 +25,7 @@ static char * tsdbTbDataGetUid(const void *arg); static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row); STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { - STsdbMemTable *pMemTable = (STsdbMemTable *)calloc(1, sizeof(*pMemTable)); + STsdbMemTable *pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable)); if (pMemTable == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -38,7 +38,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { pMemTable->nRow = 0; pMemTable->pMA = pTsdb->pmaf->create(pTsdb->pmaf); if (pMemTable->pMA == NULL) { - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -47,7 +47,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { tSkipListCreate(5, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t), tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid); if (pMemTable->pSlIdx == NULL) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -55,7 +55,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { if (pMemTable->pHashIdx == NULL) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); tSkipListDestroy(pMemTable->pSlIdx); - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -69,7 +69,7 @@ void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) { if (pMemTable->pMA) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); } - free(pMemTable); + taosMemoryFree(pMemTable); } } @@ -227,7 +227,35 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey return 0; } -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) { + ASSERT(pMsg != NULL); + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + + terrno = TSDB_CODE_SUCCESS; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + while (true) { + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (pBlock == NULL) break; + + pBlock->uid = htobe64(pBlock->uid); + pBlock->suid = htobe64(pBlock->suid); + pBlock->sversion = htonl(pBlock->sversion); + pBlock->dataLen = htonl(pBlock->dataLen); + pBlock->schemaLen = htonl(pBlock->schemaLen); + pBlock->numOfRows = htons(pBlock->numOfRows); + } + + if (terrno != TSDB_CODE_SUCCESS) return -1; + return 0; +} + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; @@ -248,7 +276,7 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { if (pBlock == NULL) break; pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); + pBlock->suid = htobe64(pBlock->suid); pBlock->sversion = htonl(pBlock->sversion); pBlock->dataLen = htonl(pBlock->dataLen); pBlock->schemaLen = htonl(pBlock->schemaLen); @@ -376,7 +404,7 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p } static STbData *tsdbNewTbData(tb_uid_t uid) { - STbData *pTbData = (STbData *)calloc(1, sizeof(*pTbData)); + STbData *pTbData = (STbData *)taosMemoryCalloc(1, sizeof(*pTbData)); if (pTbData == NULL) { return NULL; } @@ -397,14 +425,14 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { // tkeyComparFn, skipListCreateFlags, tsdbGetTsTupleKey); // if (pTableData->pData == NULL) { // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - // free(pTableData); + // taosMemoryFree(pTableData); // return NULL; // } pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTbData->pData == NULL) { - free(pTbData); + taosMemoryFree(pTbData); return NULL; } @@ -414,7 +442,7 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { static void tsdbFreeTbData(STbData *pTbData) { if (pTbData) { tSkipListDestroy(pTbData->pData); - free(pTbData); + taosMemoryFree(pTbData); } } @@ -541,7 +569,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { } } if (addNew) { - int code = pthread_cond_signal(&pBufPool->poolNotEmpty); + int code = taosThreadCondSignal(&pBufPool->poolNotEmpty); if (code != 0) { if (tsdbUnlockRepo(pRepo) < 0) return -1; tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); @@ -582,7 +610,7 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot, SArray *pATab pSnapshot->mem = &(pSnapshot->mtable); - pSnapshot->mem->tData = (STableData **)calloc(pSnapshot->omem->maxTables, sizeof(STableData *)); + pSnapshot->mem->tData = (STableData **)taosMemoryCalloc(pSnapshot->omem->maxTables, sizeof(STableData *)); if (pSnapshot->mem->tData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; taosRUnLockLatch(&(pSnapshot->omem->latch)); @@ -629,7 +657,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot) { tsdbFreeTableData(pTableData); } } - tfree(pSnapshot->mem->tData); + taosMemoryFreeClear(pSnapshot->mem->tData); tsdbUnRefMemTable(pRepo, pSnapshot->omem); } @@ -990,10 +1018,10 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, STSRow* ro TSDB_WLOCK_TABLE(pTable); SDataCol *pDataCol = &(pLatestCols[idx]); if (pDataCol->pData == NULL) { - pDataCol->pData = malloc(pTCol->bytes); + pDataCol->pData = taosMemoryMalloc(pTCol->bytes); pDataCol->bytes = pTCol->bytes; } else if (pDataCol->bytes < pTCol->bytes) { - pDataCol->pData = realloc(pDataCol->pData, pTCol->bytes); + pDataCol->pData = taosMemoryRealloc(pDataCol->pData, pTCol->bytes); pDataCol->bytes = pTCol->bytes; } // the actual value size diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6b2857c4110cb6121a7be585dd383873a300cc54..bac5255d17fd8de08dc19a0060321630b5c3caff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tsdbDef.h" #include #include "os.h" #include "talgo.h" @@ -20,7 +21,6 @@ #include "tdataformat.h" #include "texception.h" #include "tsdb.h" -#include "tsdbDef.h" #include "tsdbFS.h" #include "tsdbLog.h" #include "tsdbReadImpl.h" @@ -355,7 +355,7 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* } static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, uint64_t taskId) { - STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle)); + STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle)); if (pReadHandle == NULL) { goto _end; } @@ -388,7 +388,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file - pReadHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); + pReadHandle->statis = taosMemoryCalloc(pCond->numOfCols, sizeof(SDataStatis)); if (pReadHandle->statis == NULL) { goto _end; } @@ -401,10 +401,10 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, for (int32_t i = 0; i < pCond->numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCond->colList[i]; - colInfo.pData = calloc(1, EXTRA_BYTES + pReadHandle->outputCapacity * pCond->colList[i].bytes); - if (colInfo.pData == NULL) { + + int32_t code = blockDataEnsureColumnCapacity(&colInfo, pReadHandle->outputCapacity); + if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -597,7 +597,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr assert(pGroupList); size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); - STableGroupInfo* pNew = calloc(1, sizeof(STableGroupInfo)); + STableGroupInfo* pNew = taosMemoryCalloc(1, sizeof(STableGroupInfo)); pNew->pGroupList = taosArrayInit(numOfGroup, POINTER_BYTES); for(int32_t i = 0; i < numOfGroup; ++i) { @@ -1009,7 +1009,7 @@ static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, i if (pCheckInfo->compSize < (int32_t)compIndex->len) { assert(compIndex->len > 0); - char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); + char* t = taosMemoryRealloc(pCheckInfo->pCompInfo, compIndex->len); if (t == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; code = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1377,7 +1377,6 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { } static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { - char* pData = NULL; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; @@ -1403,43 +1402,38 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t continue; } - int32_t bytes = pColInfo->info.bytes; - - if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; - } - if (!isAllRowsNull(src) && pColInfo->info.colId == src->colId) { - if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) { - memmove(pData, (char*)src->pData + bytes * start, bytes * num); - } else { // handle the var-string - char* dst = pData; + if (!IS_VAR_DATA_TYPE(pColInfo->info.type)) { // todo opt performance +// memmove(pData, (char*)src->pData + bytes * start, bytes * num); + for(int32_t k = start; k < num + start; ++k) { + SCellVal sVal = {0}; + if (tdGetColDataOfRow(&sVal, src, k) < 0) { + TASSERT(0); + } + if (sVal.valType == TD_VTYPE_NULL) { + colDataAppend(pColInfo, k, NULL, true); + } else { + colDataAppend(pColInfo, k, sVal.val, false); + } + } + } else { // handle the var-string // todo refactor, only copy one-by-one for (int32_t k = start; k < num + start; ++k) { - SCellVal sVal = {0}; + SCellVal sVal = {0}; if(tdGetColDataOfRow(&sVal, src, k) < 0){ TASSERT(0); } - memcpy(dst, sVal.val, varDataTLen(sVal.val)); - dst += bytes; + + colDataAppend(pColInfo, k, sVal.val, false); } } j++; i++; } else { // pColInfo->info.colId < src->colId, it is a NULL data - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - char* dst = pData; - - for(int32_t k = start; k < num + start; ++k) { - setVardataNull(dst, pColInfo->info.type); - dst += bytes; - } - } else { - setNullN(pData, pColInfo->info.type, pColInfo->info.bytes, num); + for(int32_t k = start; k < num + start; ++k) { // TODO opt performance + colDataAppend(pColInfo, k, NULL, true); } i++; } @@ -1447,23 +1441,9 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t while (i < requiredNumOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; - } - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - char* dst = pData; - - for(int32_t k = start; k < num + start; ++k) { - setVardataNull(dst, pColInfo->info.type); - dst += pColInfo->info.bytes; - } - } else { - setNullN(pData, pColInfo->info.type, pColInfo->info.bytes, num); + for(int32_t k = start; k < num + start; ++k) { + colDataAppend(pColInfo, k, NULL, true); // TODO add a fast version to set a number of consecutive NULL value. } - i++; } @@ -1473,14 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t return numOfRows + num; } -// TODO fix bug for reverse copy data -// TODO handle the null data +// TODO fix bug for reverse copy data problem // Note: row1 always has high priority static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1, STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) { #if 1 - char* pData = NULL; STSchema* pSchema; STSRow* row; int16_t colId; @@ -1522,12 +1500,6 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } - int32_t colIdOfRow1; if(j >= numOfColsOfRow1) { colIdOfRow1 = INT32_MAX; @@ -1590,43 +1562,11 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit if (colId == pColInfo->info.colId) { if (tdValTypeIsNorm(sVal.valType)) { - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, sVal.val, varDataTLen(sVal.val)); - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)sVal.val; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)sVal.val; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)sVal.val; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)sVal.val; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, sVal.val); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, sVal.val); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - *(TSKEY*)pData = *(TSKEY*)sVal.val; - break; - default: - memcpy(pData, sVal.val, pColInfo->info.bytes); - } + colDataAppend(pColInfo, numOfRows, sVal.val, false); } else if (forceSetNull) { colDataAppend(pColInfo, numOfRows, NULL, true); } + i++; if(row == row1) { @@ -2034,15 +1974,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { } static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { - tfree(pSupporter->numOfBlocksPerTable); - tfree(pSupporter->blockIndexArray); + taosMemoryFreeClear(pSupporter->numOfBlocksPerTable); + taosMemoryFreeClear(pSupporter->blockIndexArray); for (int32_t i = 0; i < numOfTables; ++i) { STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i]; - tfree(pBlockInfo); + taosMemoryFreeClear(pBlockInfo); } - tfree(pSupporter->pDataBlockInfo); + taosMemoryFreeClear(pSupporter->pDataBlockInfo); } static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { @@ -2081,7 +2021,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu if (pTsdbReadHandle->allocSize < size) { pTsdbReadHandle->allocSize = (int32_t)size; - char* tmp = realloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); + char* tmp = taosMemoryRealloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); if (tmp == NULL) { return TSDB_CODE_TDB_OUT_OF_MEMORY; } @@ -2097,9 +2037,9 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; - sup.numOfBlocksPerTable = calloc(1, sizeof(int32_t) * numOfTables); - sup.blockIndexArray = calloc(1, sizeof(int32_t) * numOfTables); - sup.pDataBlockInfo = calloc(1, POINTER_BYTES * numOfTables); + sup.numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + sup.blockIndexArray = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + sup.pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); if (sup.numOfBlocksPerTable == NULL || sup.blockIndexArray == NULL || sup.pDataBlockInfo == NULL) { cleanBlockOrderSupporter(&sup, 0); @@ -2118,7 +2058,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu SBlock* pBlock = pTableCheck->pCompInfo->blocks; sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks; - char* buf = malloc(sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); + char* buf = taosMemoryMalloc(sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); if (buf == NULL) { cleanBlockOrderSupporter(&sup, numOfQualTables); return TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -2188,7 +2128,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu tsdbDebug("%p %d data blocks sort completed, %s", pTsdbReadHandle, cnt, pTsdbReadHandle->idStr); cleanBlockOrderSupporter(&sup, numOfTables); - free(pTree); + taosMemoryFree(pTree); return TSDB_CODE_SUCCESS; } @@ -2596,12 +2536,12 @@ static void destroyHelper(void* param) { // tQueryInfo* pInfo = (tQueryInfo*)param; // if (pInfo->optr != TSDB_RELATION_IN) { -// tfree(pInfo->q); +// taosMemoryFreeClear(pInfo->q); // } else { // taosHashCleanup((SHashObj *)(pInfo->q)); // } - free(param); + taosMemoryFree(param); } #define TSDB_PREV_ROW 0x1 @@ -2676,7 +2616,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // return false; // } mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->tableId, NULL, NULL, true); - tfree(pRow); + taosMemoryFreeClear(pRow); // update the last key value pCheckInfo->lastKey = key + step; @@ -2964,7 +2904,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // // SColumnInfoData colInfo = {{0}, 0}; // colInfo.info = pCol->info; -// colInfo.pData = calloc(1, pCol->info.bytes); +// colInfo.pData = taosMemoryCalloc(1, pCol->info.bytes); // if (colInfo.pData == NULL) { // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // goto out_of_memory; @@ -2983,7 +2923,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MAX}; // } // -// cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); +// cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); // if (cond.colList == NULL) { // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // goto out_of_memory; @@ -2995,7 +2935,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // } // // pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->idStr, pMemRef); -// tfree(cond.colList); +// taosMemoryFreeClear(cond.colList); // // // current table, only one table // STableCheckInfo* pCurrent = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); @@ -3354,7 +3294,7 @@ void filterPrepare(void* expr, void* param) { return; } - pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); + pExpr->_node.info = taosMemoryCalloc(1, sizeof(tQueryInfo)); STSchema* pTSSchema = (STSchema*) param; tQueryInfo* pInfo = pExpr->_node.info; @@ -3387,7 +3327,7 @@ void filterPrepare(void* expr, void* param) { size = pSchema->bytes; } // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(TdUcs4) space. - pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + pInfo->q = taosMemoryCalloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); tVariantDump(pCond, pInfo->q, pSchema->type, true); } } @@ -3673,7 +3613,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch // if (tagExpr != NULL) { // CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, tagExpr, NULL); // tExprNode* tbnameExpr = expr; -// expr = calloc(1, sizeof(tExprNode)); +// expr = taosMemoryCalloc(1, sizeof(tExprNode)); // if (expr == NULL) { // THROW( TSDB_CODE_TDB_OUT_OF_MEMORY ); // } @@ -3787,7 +3727,7 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) { size_t cols = taosArrayGetSize(pColumnInfoData); for (int32_t i = 0; i < cols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pColumnInfoData, i); - tfree(pColInfo->pData); + taosMemoryFreeClear(pColInfo->pData); } taosArrayDestroy(pColumnInfoData); @@ -3800,7 +3740,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i); destroyTableMemIterator(p); - tfree(p->pCompInfo); + taosMemoryFreeClear(p->pCompInfo); } taosArrayDestroy(pTableCheckInfo); @@ -3817,8 +3757,8 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn); - tfree(pTsdbReadHandle->pDataBlockInfo); - tfree(pTsdbReadHandle->statis); + taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); + taosMemoryFreeClear(pTsdbReadHandle->statis); if (!emptyQueryTimewindow(pTsdbReadHandle)) { // tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); @@ -3843,7 +3783,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %s", pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->idStr); - tfree(pTsdbReadHandle); + taosMemoryFreeClear(pTsdbReadHandle); } #if 0 @@ -3902,15 +3842,15 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { - pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->start->optr = queryColInfo->optr; pCond->start->v = queryColInfo->q; } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - pCond->end = calloc(1, sizeof(SEndPoint)); + pCond->end = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->end->optr = queryColInfo->optr; pCond->end->v = queryColInfo->q; } else if (optr == TSDB_RELATION_IN) { - pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->start->optr = queryColInfo->optr; pCond->start->v = queryColInfo->q; } else if (optr == TSDB_RELATION_LIKE) { @@ -4056,8 +3996,8 @@ static void queryIndexedColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SAr } } - free(cond.start); - free(cond.end); + taosMemoryFree(cond.start); + taosMemoryFree(cond.end); tSkipListDestroyIter(iter); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index f6827eaae187dfb3da31110c81dcf38c24a67702..9619ac036efa63a3b2aaa64855ca2f52600f0a72 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -727,9 +727,9 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * } ASSERT(pBlockCol->colId == pDataCol->colId); - // set the bitmap - pDataCol->bitmap = pBlockCol->bitmap; } + // set the bitmap + pDataCol->bitmap = pBlockCol->bitmap; if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 02a0b587d5793d64444fe70d6662019ce75dc07f..9d5e1327721294cb42a85f0dc9a0bef728febbfe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -25,6 +25,7 @@ static const char *TSDB_SMA_DNAME[] = { #define SMA_STORAGE_TSDB_TIMES 10 #define SMA_STORAGE_SPLIT_HOURS 24 #define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8 +#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds #define SMA_STATE_HASH_SLOT 4 #define SMA_STATE_ITEM_HASH_SLOT 32 @@ -37,7 +38,7 @@ typedef enum { } ESmaStorageLevel; typedef struct { - STsdb * pTsdb; + STsdb *pTsdb; SDBFile dFile; int32_t interval; // interval with the precision of DB } STSmaWriteH; @@ -48,7 +49,7 @@ typedef struct { } SmaFsIter; typedef struct { - STsdb * pTsdb; + STsdb *pTsdb; SDBFile dFile; int32_t interval; // interval with the precision of DB int32_t blockSize; // size of SMA block item @@ -60,14 +61,15 @@ typedef struct { typedef struct { /** * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. - * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, - * without information about its previous state. * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from * Streaming Module or TSDB local persistence. + * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, + * without information about its previous state. + * - TSDB_SMA_STAT_DROPPED: 1)sma dropped */ int8_t state; // ETsdbSmaStat SHashObj *expiredWindows; // key: skey of time window, value: N/A - STSma * pSma; + STSma *pSma; // cache schema } SSmaStatItem; struct SSmaStat { @@ -78,11 +80,13 @@ struct SSmaStat { // declaration of static functions // expired window -static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg); +static int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg); +static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); +static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path); -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv); +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did); +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv); static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); @@ -102,31 +106,80 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid); -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey); +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid); +static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); +// mgmt interface +static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); + // implementation +static FORCE_INLINE int8_t tsdbSmaStat(SSmaStatItem *pStatItem) { + if (pStatItem) { + return atomic_load_8(&pStatItem->state); + } + return TSDB_SMA_STAT_UNKNOWN; +} + +static FORCE_INLINE bool tsdbSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { + if (!pStatItem) { + return false; + } + + if (state) { + *state = atomic_load_8(&pStatItem->state); + return *state == TSDB_SMA_STAT_OK; + } + return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; +} + +static FORCE_INLINE bool tsdbSmaStatIsExpired(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; +} + +static FORCE_INLINE bool tsdbSmaStatIsDropped(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; +} + +static FORCE_INLINE void tsdbSmaStatSetOK(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); + } +} + +static FORCE_INLINE void tsdbSmaStatSetExpired(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); + } +} + +static FORCE_INLINE void tsdbSmaStatSetDropped(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); + } +} + static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vgId, TSDB_SMA_DNAME[smaType]); + snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP, + TSDB_SMA_DNAME[smaType]); } -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) { SSmaEnv *pEnv = NULL; - pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv)); + pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); if (pEnv == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - int code = pthread_rwlock_init(&(pEnv->lock), NULL); + int code = taosThreadRwlockInit(&(pEnv->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); - free(pEnv); + taosMemoryFree(pEnv); return NULL; } @@ -137,12 +190,16 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return NULL; } + pEnv->did = did; + if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) { tsdbFreeSmaEnv(pEnv); return NULL; } - if (tsdbOpenBDBEnv(&pEnv->dbEnv, pEnv->path) != TSDB_CODE_SUCCESS) { + char aname[TSDB_FILENAME_LEN] = {0}; + tfsAbsoluteName(pTsdb->pTfs, did, path, aname); + if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { tsdbFreeSmaEnv(pEnv); return NULL; } @@ -150,14 +207,14 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return pEnv; } -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) { +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv) { if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } if (*pEnv == NULL) { - if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) { + if ((*pEnv = tsdbNewSmaEnv(pTsdb, path, did)) == NULL) { return TSDB_CODE_FAILED; } } @@ -174,16 +231,16 @@ static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) { void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { if (pSmaEnv) { tsdbDestroySmaState(pSmaEnv->pStat); - tfree(pSmaEnv->pStat); - tfree(pSmaEnv->path); - pthread_rwlock_destroy(&(pSmaEnv->lock)); + taosMemoryFreeClear(pSmaEnv->pStat); + taosMemoryFreeClear(pSmaEnv->path); + taosThreadRwlockDestroy(&(pSmaEnv->lock)); tsdbCloseBDBEnv(pSmaEnv->dbEnv); } } void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) { tsdbDestroySmaEnv(pSmaEnv); - tfree(pSmaEnv); + taosMemoryFreeClear(pSmaEnv); return NULL; } @@ -215,7 +272,7 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { * tsdbInitSmaStat invoked in other multithread environment later. */ if (*pSmaStat == NULL) { - *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat)); + *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat)); if (*pSmaStat == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; @@ -225,7 +282,7 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if ((*pSmaStat)->smaStatItems == NULL) { - tfree(*pSmaStat); + taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; } } @@ -235,18 +292,28 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { SSmaStatItem *pItem = NULL; - pItem = (SSmaStatItem *)calloc(1, sizeof(SSmaStatItem)); + pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); if (pItem) { pItem->state = state; pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), true, HASH_ENTRY_LOCK); if (!pItem->expiredWindows) { - tfree(pItem); + taosMemoryFreeClear(pItem); } } return pItem; } +static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { + if (pSmaStatItem != NULL) { + tdDestroyTSma(pSmaStatItem->pSma); + taosMemoryFreeClear(pSmaStatItem->pSma); + taosHashCleanup(pSmaStatItem->expiredWindows); + taosMemoryFreeClear(pSmaStatItem); + } + return NULL; +} + /** * @brief Release resources allocated for its member fields, not including itself. * @@ -259,12 +326,7 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { void *item = taosHashIterate(pSmaStat->smaStatItems, NULL); while (item != NULL) { SSmaStatItem *pItem = *(SSmaStatItem **)item; - if (pItem != NULL) { - tdDestroyTSma(pItem->pSma); - tfree(pItem->pSma); - taosHashCleanup(pItem->expiredWindows); - tfree(pItem); - } + tsdbFreeSmaStatItem(pItem); item = taosHashIterate(pSmaStat->smaStatItems, item); } taosHashCleanup(pSmaStat->smaStatItems); @@ -296,7 +358,6 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&pTsdb->pTSmaEnv) : atomic_load_ptr(&pTsdb->pRSmaEnv); if (pEnv == NULL) { char rname[TSDB_FILENAME_LEN] = {0}; - char aname[TSDB_FILENAME_LEN] = {0}; // use TSDB_FILENAME_LEN currently SDiskID did = {0}; tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did); @@ -305,14 +366,13 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { return TSDB_CODE_FAILED; } tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname); - tfsAbsoluteName(pTsdb->pTfs, did, rname, aname); if (tfsMkdirRecurAt(pTsdb->pTfs, rname, did) != TSDB_CODE_SUCCESS) { tsdbUnlockRepo(pTsdb); return TSDB_CODE_FAILED; } - if (tsdbInitSmaEnv(pTsdb, aname, &pEnv) != TSDB_CODE_SUCCESS) { + if (tsdbInitSmaEnv(pTsdb, rname, did, &pEnv) != TSDB_CODE_SUCCESS) { tsdbUnlockRepo(pTsdb); return TSDB_CODE_FAILED; } @@ -325,26 +385,78 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { return TSDB_CODE_SUCCESS; }; + +static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey) { + SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + if (pItem == NULL) { + // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later + pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state + if (pItem == NULL) { + // Response to stream computing: OOM + // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. + return TSDB_CODE_FAILED; + } + + // cache smaMeta + STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid); + if (pSma == NULL) { + terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, + tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pSma = pSma; + + if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { + // If error occurs during put smaStatItem, free the resources of pItem + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + return TSDB_CODE_FAILED; + } + } else if ((pItem = *(SSmaStatItem **)pItem) == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + int8_t state = TSDB_SMA_STAT_EXPIRED; + if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &state, sizeof(state)) != 0) { + // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would + // tell query module to query raw TS data. + // N.B. + // 1) It is assumed to be extemely little probability event of fail to taosHashPut. + // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired + // windows failed to put into hash table. + taosHashCleanup(pItem->expiredWindows); + taosMemoryFreeClear(pItem->pSma); + taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); + return TSDB_CODE_FAILED; + } + tsdbDebug("vgId:%d smaIndex %" PRIi64 " tsKey %" PRIi64 " is put to hash", REPO_ID(pTsdb), indexUid, winSKey); +} + /** * @brief Update expired window according to msg from stream computing module. * * @param pTsdb - * @param smaType ETsdbSmaType - * @param msg + * @param msg SSubmitReq * @return int32_t */ -int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) { - if (!msg || !pTsdb->pMeta) { - terrno = TSDB_CODE_INVALID_PTR; +int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg) { + const SSubmitReq *pMsg = (const SSubmitReq *)msg; + + if (pMsg->length <= sizeof(SSubmitReq)) { + terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return TSDB_CODE_FAILED; } - - if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; + if (!pTsdb->pMeta) { + terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } - // TODO: decode the msg from Stream Computing module => start +// TODO: decode the msg from Stream Computing module => start +#ifdef TSDB_SMA_TESTx int64_t indexUid = SMA_TEST_INDEX_UID; const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; @@ -352,70 +464,101 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) { for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { expiredWindows[i] = skey1 + i; } +#else + +#endif // TODO: decode the msg <= end - SSmaEnv * pEnv = REPO_SMA_ENV(pTsdb, smaType); + if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + +#ifndef TSDB_SMA_TEST + TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; +#endif + + + // Firstly, assume that tSma can only be created on super table/normal table. + // getActiveTimeWindow + + + SSmaEnv *pEnv = REPO_SMA_ENV(pTsdb, TSDB_SMA_TYPE_TIME_RANGE); SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL); + + // basic procedure + // TODO: optimization tsdbRefSmaStat(pTsdb, pStat); - SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (pItem == NULL) { - pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state - if (pItem == NULL) { - // Response to stream computing: OOM - // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - // cache smaMeta - STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid); - if (pSma == NULL) { - terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; - taosHashCleanup(pItem->expiredWindows); - free(pItem); - tsdbUnRefSmaStat(pTsdb, pStat); - tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, - tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - pItem->pSma = pSma; + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SInterval interval = {0}; - if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { - // If error occurs during put smaStatItem, free the resources of pItem - taosHashCleanup(pItem->expiredWindows); - free(pItem); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } + if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; } - int8_t state = TSDB_SMA_STAT_EXPIRED; - for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { - if (taosHashPut(pItem->expiredWindows, expiredWindows + i, sizeof(TSKEY), &state, sizeof(state)) != 0) { - // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would - // tell query module to query raw TS data. - // N.B. - // 1) It is assumed to be extemely little probability event of fail to taosHashPut. - // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired - // windows failed to put into hash table. - taosHashCleanup(pItem->expiredWindows); - tfree(pItem->pSma); - taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; + while (true) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (pBlock == NULL) break; + + STSmaWrapper *pSW = NULL; + STSma *pTSma = NULL; + + SSubmitBlkIter blkIter = {0}; + if (tInitSubmitBlkIter(pBlock, &blkIter) != TSDB_CODE_SUCCESS) { + tdFreeTSmaWrapper(pSW); + break; + } + + while (true) { + STSRow *row = tGetSubmitBlkNext(&blkIter); + if (row == NULL) { + tdFreeTSmaWrapper(pSW); + break; + } + if(pSW == NULL) { + if((pSW =metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) { + break; + } + if((pSW->number) <= 0 || (pSW->tSma == NULL)) { + tdFreeTSmaWrapper(pSW); + break; + } + pTSma = pSW->tSma; + } + + interval.interval = pTSma->interval; + interval.intervalUnit = pTSma->intervalUnit; + interval.offset = pTSma->offset; + interval.precision = REPO_CFG(pTsdb)->precision; + interval.sliding = pTSma->sliding; + interval.slidingUnit = pTSma->slidingUnit; + + TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); + + tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey); } - tsdbDebug("vgId:%d smaIndex %" PRIi64 " tsKey %" PRIi64 " is put to hash", REPO_ID(pTsdb), indexUid, - expiredWindows[i]); } tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_SUCCESS; } +/** + * @brief When sma data received from stream computing, make the relative expired window valid. + * + * @param pTsdb + * @param pStat + * @param indexUid + * @param skey + * @return int32_t + */ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { SSmaStatItem *pItem = NULL; @@ -434,6 +577,15 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind skey, indexUid); return TSDB_CODE_FAILED; } + // TODO: use a standalone interface to received state upate notification from stream computing module. + /** + * @brief state + * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. + * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. + * when batch data caculation not finised) + * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. + */ + pItem->state = TSDB_SMA_STAT_OK; } else { // error handling tsdbUnRefSmaStat(pTsdb, pStat); @@ -514,7 +666,7 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t k #ifdef _TEST_SMA_PRINT_DEBUG_LOG_ uint32_t valueSize = 0; - void * data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); + void *data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); ASSERT(data != NULL); for (uint32_t v = 0; v < valueSize; v += 8) { tsdbWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); @@ -631,7 +783,7 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p assert(pTbData->dataLen > 0); STSmaColData *pColData = (STSmaColData *)POINTER_SHIFT(pTbData->data, tbLen); char smaKey[SMA_KEY_LEN] = {0}; - void * pSmaKey = &smaKey; + void *pSmaKey = &smaKey; #if 0 printf("tsdbInsertTSmaDataSection: index %" PRIi64 ", skey %" PRIi64 " table[%" PRIi64 "]col[%" PRIu16 "]\n", pData->indexUid, pData->skey, pTbData->tableUid, pColData->colId); @@ -660,14 +812,13 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { } } -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) { +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL); pSmaH->dFile.fid = fid; - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid); + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); pSmaH->dFile.path = strdup(tSmaFile); return TSDB_CODE_SUCCESS; @@ -706,10 +857,12 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe * @return int32_t */ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg * pCfg = REPO_CFG(pTsdb); + STsdbCfg *pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; - if (!atomic_load_ptr(&pTsdb->pTSmaEnv)) { + if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return terrno; @@ -727,6 +880,32 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return TSDB_CODE_FAILED; } + SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv); + SSmaStatItem *pItem = NULL; + + tsdbRefSmaStat(pTsdb, pStat); + + if (pStat && pStat->smaStatItems) { + pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + } + + if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + } + // Step 1: Judge the storage level and days int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); @@ -735,17 +914,19 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file // - Set and open the DFile or the B+Tree file // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); + tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } // TODO:tsdbEndTSmaCommit(); @@ -754,9 +935,59 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; } +/** + * @brief Drop tSma data and local cache + * - insert/query reference + * @param pTsdb + * @param msg + * @return int32_t + */ +static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + + // clear local cache + if (pEnv) { + tsdbDebug("vgId:%d drop tSma local cache for %" PRIi64, REPO_ID(pTsdb), indexUid); + + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if ((pItem != NULL) || ((pItem = *(SSmaStatItem **)pItem) != NULL)) { + if (tsdbSmaStatIsDropped(pItem)) { + tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + + tsdbWLockSma(pEnv); + if (tsdbSmaStatIsDropped(pItem)) { + tsdbUnLockSma(pEnv); + tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + tsdbSmaStatSetDropped(pItem); + tsdbUnLockSma(pEnv); + + int32_t nSleep = 0; + while (true) { + if (T_REF_VAL_GET(SMA_ENV_STAT(pEnv)) <= 0) { + break; + } + taosSsleep(1); + if (++nSleep > SMA_DROP_EXPIRED_TIME) { + break; + }; + } + + tsdbFreeSmaStatItem(pItem); + tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", REPO_ID(pTsdb), indexUid); + } + } + // clear sma data files + // TODO: +} + static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; @@ -768,37 +999,66 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, } static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg * pCfg = REPO_CFG(pTsdb); + STsdbCfg *pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - STSmaWriteH tSmaH = {0}; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); - tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData); + if (pEnv == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + return terrno; + } if (pData->dataLen <= 0) { TASSERT(0); terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return TSDB_CODE_FAILED; } - // Step 1: Judge the storage level - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); - int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile; + STSmaWriteH tSmaH = {0}; - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file - // - Set and open the DFile or the B+Tree file + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + return TSDB_CODE_FAILED; + } + int64_t indexUid = SMA_TEST_INDEX_UID; + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + } + + // Step 1: Judge the storage level and days + int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); - // Save all the TSma data to one file + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file + // - Set and open the DFile or the B+Tree file // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); + tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); + if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { + tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + return TSDB_CODE_FAILED; + } - tsdbInsertTSmaDataSection(&tSmaH, pData); + if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { + tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + return TSDB_CODE_FAILED; + } // TODO:tsdbEndTSmaCommit(); - // reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey); + // Step 3: reset the SSmaStat + tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); + tsdbDestroyTSmaWriteH(&tSmaH); return TSDB_CODE_SUCCESS; } @@ -822,13 +1082,16 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interv * @brief Init of tSma FS * * @param pReadH + * @param indexUid * @param skey * @return int32_t */ -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pSmaH->pTsdb)->precision)); +static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { + STsdb *pTsdb = pSmaH->pTsdb; + + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pTsdb)->precision)); char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pSmaH->pTsdb), fid); + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); pSmaH->dFile.path = strdup(tSmaFile); pSmaH->smaFsIter.iter = 0; pSmaH->smaFsIter.fid = fid; @@ -887,14 +1150,16 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) { - if (!atomic_load_ptr(&pTsdb->pTSmaEnv)) { + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + + if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } - tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid)); + tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pEnv)); + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) { // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if // it's NULL. @@ -915,6 +1180,15 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ #endif #if 1 + int8_t smaStat = 0; + if (!tsdbSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query + tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, REPO_ID(pTsdb), indexUid, + tstrerror(terrno), smaStat); + return TSDB_CODE_FAILED; + } + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY)) != NULL) { // TODO: mark this window as expired. tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb), @@ -926,11 +1200,12 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); #endif + STSmaReadH tReadH = {0}; tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit); tsdbCloseDBF(&tReadH.dFile); - tsdbInitTSmaFile(&tReadH, querySKey); + tsdbInitTSmaFile(&tReadH, indexUid, querySKey); if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) { tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); return TSDB_CODE_FAILED; @@ -944,7 +1219,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), SMA_KEY_LEN); - void * result = NULL; + void *result = NULL; uint32_t valueSize = 0; if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) { tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 @@ -960,7 +1235,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tsdbWarn("vgId:%d get sma data v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); } #endif - tfree(result); // TODO: fill the result to output + taosMemoryFreeClear(result); // TODO: fill the result to output #if 0 int32_t nResult = 0; @@ -1026,15 +1301,8 @@ int32_t tsdbRemoveTSmaData(STsdb *pTsdb, void *smaIndex, STimeWindow *pWin) { } #endif -/** - * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine - * - * @param pTsdb - * @param param - * @param msg - * @return int32_t - * TODO: Who is responsible for resource allocate and release? - */ + +// TODO: Who is responsible for resource allocate and release? int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tsdbInsertTSmaDataImpl(pTsdb, msg)) < 0) { @@ -1043,21 +1311,14 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { return code; } -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) { +int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, const char *msg) { int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) { + if ((code = tsdbUpdateExpiredWindowImpl(pTsdb, msg)) < 0) { tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; } -/** - * @brief Insert Time-range-wise Rollup Sma(RSma) data - * - * @param pTsdb - * @param msg - * @return int32_t - */ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tsdbInsertRSmaDataImpl(pTsdb, msg)) < 0) { @@ -1066,6 +1327,7 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { return code; } + int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) { int32_t code = TSDB_CODE_SUCCESS; @@ -1074,4 +1336,13 @@ int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; +} + + +int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbDropTSmaDataImpl(pTsdb, indexUid)) < 0) { + tsdbWarn("vgId:%d drop tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 3ccb483fe48c459b52dcd783ec62ee45ad9e4112..5590f13cc6e9c602a5530910de3c0154d547219f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -32,48 +32,4 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { } } return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL); -} - -#if 0 -/** - * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine - * - * @param pTsdb - * @param param - * @param msg - * @return int32_t - * TODO: Who is responsible for resource allocate and release? - */ -int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertTSmaDataImpl(pTsdb, msg)) < 0) { - tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) { - tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -/** - * @brief Insert Time-range-wise Rollup Sma(RSma) data - * - * @param pTsdb - * @param param - * @param msg - * @return int32_t - */ -int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertRSmaDataImpl(pTsdb, msg)) < 0) { - tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index 14b9a5124ffcb89f91194190b661f5e70062c98f..ab0da1451dec1b818290cce00b21d4df62b1af2d 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -19,7 +19,7 @@ static SVArenaNode *vArenaNodeNew(uint64_t capacity); static void vArenaNodeFree(SVArenaNode *pNode); SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { - SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); + SVMemAllocator *pVMA = (SVMemAllocator *)taosMemoryMalloc(sizeof(*pVMA)); if (pVMA == NULL) { return NULL; } @@ -31,7 +31,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->pNode = vArenaNodeNew(capacity); if (pVMA->pNode == NULL) { - free(pVMA); + taosMemoryFree(pVMA); return NULL; } @@ -48,7 +48,7 @@ void vmaDestroy(SVMemAllocator *pVMA) { vArenaNodeFree(pNode); } - free(pVMA); + taosMemoryFree(pVMA); } } @@ -99,7 +99,7 @@ bool vmaIsFull(SVMemAllocator *pVMA) { static SVArenaNode *vArenaNodeNew(uint64_t capacity) { SVArenaNode *pNode = NULL; - pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity); + pNode = (SVArenaNode *)taosMemoryMalloc(sizeof(*pNode) + capacity); if (pNode == NULL) { return NULL; } @@ -112,6 +112,6 @@ static SVArenaNode *vArenaNodeNew(uint64_t capacity) { static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { - free(pNode); + taosMemoryFree(pNode); } } diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 9d1877bdb7dc3b5b4b0b71ea92e85d2056fc60c5..5be88cdc2e0fc2b9871a55be8e735e06ba1ddf2f 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -19,8 +19,8 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - pthread_mutex_t mutex; - pthread_cond_t hasFree; + TdThreadMutex mutex; + TdThreadCond hasFree; TD_DLIST(SVMemAllocator) free; TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; @@ -34,7 +34,7 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocato int vnodeOpenBufPool(SVnode *pVnode) { uint64_t capacity; - if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { + if ((pVnode->pBufPool = (SVBufPool *)taosMemoryCalloc(1, sizeof(SVBufPool))) == NULL) { /* TODO */ return -1; } @@ -57,7 +57,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); } - pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)malloc(sizeof(SMemAllocatorFactory)); + pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)taosMemoryMalloc(sizeof(SMemAllocatorFactory)); if (pVnode->pBufPool->pMAF == NULL) { // TODO: handle error return -1; @@ -71,7 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { void vnodeCloseBufPool(SVnode *pVnode) { if (pVnode->pBufPool) { - tfree(pVnode->pBufPool->pMAF); + taosMemoryFreeClear(pVnode->pBufPool->pMAF); vmaDestroy(pVnode->pBufPool->inuse); while (true) { @@ -88,7 +88,7 @@ void vnodeCloseBufPool(SVnode *pVnode) { vmaDestroy(pVMA); } - free(pVnode->pBufPool); + taosMemoryFree(pVnode->pBufPool); pVnode->pBufPool = NULL; } } @@ -161,7 +161,7 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) { SVnode * pVnode = (SVnode *)(pMAF->impl); SVMAWrapper * pWrapper; - pMA = (SMemAllocator *)calloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); + pMA = (SMemAllocator *)taosMemoryCalloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); if (pMA == NULL) { return NULL; } @@ -182,7 +182,7 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) { SVnode * pVnode = pWrapper->pVnode; SVMemAllocator *pVMA = pWrapper->pVMA; - free(pMA); + taosMemoryFree(pMA); if (--pVMA->_ref.val == 0) { TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); vmaReset(pVMA); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 727a4b41f7b687680d9c39e965f08553937bc0b1..3b7206e64d7e88f6e0aabf97be902456cc6be23e 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -32,4 +32,26 @@ int vnodeValidateOptions(const SVnodeCfg *pVnodeOptions) { void vnodeOptionsCopy(SVnodeCfg *pDest, const SVnodeCfg *pSrc) { memcpy((void *)pDest, (void *)pSrc, sizeof(SVnodeCfg)); -} \ No newline at end of file +} + +int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) { + uint32_t hashValue = 0; + + switch (pVnodeOptions->hashMethod) { + default: + hashValue = MurmurHash3_32(tableFName, strlen(tableFName)); + break; + } + + // TODO OPEN THIS !!!!!!! +#if 0 + if (hashValue < pVnodeOptions->hashBegin || hashValue > pVnodeOptions->hashEnd) { + terrno = TSDB_CODE_VND_HASH_MISMATCH; + return TSDB_CODE_VND_HASH_MISMATCH; + } +#endif + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index bc7a8460b8b842a0d8d008c650d7d78db438d1c0..696c5f39f62b93e2b9ecbdc7bb6af6e2afcb975f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -24,7 +24,7 @@ int vnodeAsyncCommit(SVnode *pVnode) { vnodeWaitCommit(pVnode); vnodeBufPoolSwitch(pVnode); - SVnodeTask *pTask = (SVnodeTask *)malloc(sizeof(*pTask)); + SVnodeTask *pTask = (SVnodeTask *)taosMemoryMalloc(sizeof(*pTask)); pTask->execute = vnodeCommit; // TODO pTask->arg = pVnode; // TODO diff --git a/source/dnode/vnode/src/vnd/vnodeInt.c b/source/dnode/vnode/src/vnd/vnodeInt.c index 7d0b594e956ef8a4c8c113f5ee2ee603d6941ea3..333da7a2e57864a871c0461c08a119467a7d095e 100644 --- a/source/dnode/vnode/src/vnd/vnodeInt.c +++ b/source/dnode/vnode/src/vnd/vnodeInt.c @@ -14,8 +14,8 @@ */ #define _DEFAULT_SOURCE -#include "vnd.h" #include "sync.h" +#include "vnd.h" // #include "vnodeInt.h" int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } @@ -27,7 +27,7 @@ int32_t vnodeSync(SVnode *pVnode) { return 0; } int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = pVnode->vgId; pLoad->role = TAOS_SYNC_STATE_LEADER; - pLoad->numOfTables = 500; + pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTimeSeries = 400; pLoad->totalStorage = 300; pLoad->compStorage = 200; @@ -41,6 +41,6 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { } int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vInfo("sync message is processed"); + /*vInfo("sync message is processed");*/ return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 2a3862c7cbf37de74506bdf217fe62ccdcfd6e52..91c6e4d263181ada878ba3d9cdc52771233aeb1d 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -27,7 +27,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnodeCfg cfg = defaultVnodeOptions; if (pVnodeCfg != NULL) { cfg.vgId = pVnodeCfg->vgId; - cfg.pDnode = pVnodeCfg->pDnode; + cfg.msgCb = pVnodeCfg->msgCb; cfg.pTfs = pVnodeCfg->pTfs; cfg.dbId = pVnodeCfg->dbId; cfg.hashBegin = pVnodeCfg->hashBegin; @@ -72,14 +72,14 @@ void vnodeDestroy(const char *path) { taosRemoveDir(path); } static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; - pVnode = (SVnode *)calloc(1, sizeof(*pVnode)); + pVnode = (SVnode *)taosMemoryCalloc(1, sizeof(*pVnode)); if (pVnode == NULL) { // TODO return NULL; } pVnode->vgId = pVnodeCfg->vgId; - pVnode->pDnode = pVnodeCfg->pDnode; + pVnode->msgCb = pVnodeCfg->msgCb; pVnode->pTfs = pVnodeCfg->pTfs; pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); @@ -92,8 +92,8 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { static void vnodeFree(SVnode *pVnode) { if (pVnode) { tsem_destroy(&(pVnode->canCommit)); - tfree(pVnode->path); - free(pVnode); + taosMemoryFreeClear(pVnode->path); + taosMemoryFree(pVnode); } } @@ -115,7 +115,8 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open tsdb sprintf(dir, "%s/tsdb", pVnode->path); - pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs); + pVnode->pTsdb = + tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs); if (pVnode->pTsdb == NULL) { // TODO: handle error return -1; @@ -131,7 +132,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); + pVnode->pTq = tqOpen(dir, pVnode, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 477deed8c8944fc511c8aa3f78951d2131171f40..8f7d5713ab69be0baceb80f779a1fbb23246e2bb 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -14,40 +14,33 @@ */ #include "vnd.h" +#include "tglobal.h" SVnodeMgr vnodeMgr = {.vnodeInitFlag = TD_MOD_UNINITIALIZED}; static void* loop(void* arg); -int vnodeInit(const SVnodeOpt *pOption) { +int vnodeInit() { if (TD_CHECK_AND_SET_MODE_INIT(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_INITIALIZED) { return 0; } vnodeMgr.stop = false; - vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; - vnodeMgr.sendReqToDnodeFp = pOption->sendReqToDnodeFp; // Start commit handers - if (pOption->nthreads > 0) { - vnodeMgr.nthreads = pOption->nthreads; - vnodeMgr.threads = (pthread_t*)calloc(pOption->nthreads, sizeof(pthread_t)); - if (vnodeMgr.threads == NULL) { - return -1; - } + vnodeMgr.nthreads = tsNumOfCommitThreads; + vnodeMgr.threads = taosMemoryCalloc(vnodeMgr.nthreads, sizeof(TdThread)); + if (vnodeMgr.threads == NULL) { + return -1; + } - pthread_mutex_init(&(vnodeMgr.mutex), NULL); - pthread_cond_init(&(vnodeMgr.hasTask), NULL); - TD_DLIST_INIT(&(vnodeMgr.queue)); + taosThreadMutexInit(&(vnodeMgr.mutex), NULL); + taosThreadCondInit(&(vnodeMgr.hasTask), NULL); + TD_DLIST_INIT(&(vnodeMgr.queue)); - for (uint16_t i = 0; i < pOption->nthreads; i++) { - pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); - // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); - } - } else { - // TODO: if no commit thread is set, then another mechanism should be - // given. Otherwise, it is a false. - ASSERT(0); + for (uint16_t i = 0; i < vnodeMgr.nthreads; i++) { + taosThreadCreate(&(vnodeMgr.threads[i]), NULL, loop, NULL); + // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } if (walInit() < 0) { @@ -63,59 +56,47 @@ void vnodeCleanup() { } // Stop commit handler - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); vnodeMgr.stop = true; - pthread_cond_broadcast(&(vnodeMgr.hasTask)); - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadCondBroadcast(&(vnodeMgr.hasTask)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); for (uint16_t i = 0; i < vnodeMgr.nthreads; i++) { - pthread_join(vnodeMgr.threads[i], NULL); + taosThreadJoin(vnodeMgr.threads[i], NULL); } - tfree(vnodeMgr.threads); - pthread_cond_destroy(&(vnodeMgr.hasTask)); - pthread_mutex_destroy(&(vnodeMgr.mutex)); + taosMemoryFreeClear(vnodeMgr.threads); + taosThreadCondDestroy(&(vnodeMgr.hasTask)); + taosThreadMutexDestroy(&(vnodeMgr.mutex)); } int vnodeScheduleTask(SVnodeTask* pTask) { - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); TD_DLIST_APPEND(&(vnodeMgr.queue), pTask); - pthread_cond_signal(&(vnodeMgr.hasTask)); + taosThreadCondSignal(&(vnodeMgr.hasTask)); - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); return 0; } -int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { - if (pVnode == NULL || pVnode->pDnode == NULL || vnodeMgr.putReqToVQueryQFp == NULL) { - terrno = TSDB_CODE_VND_APP_ERROR; - return -1; - } - return (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); -} - -void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { - (*vnodeMgr.sendReqToDnodeFp)(pVnode->pDnode, epSet, pReq); -} - /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { setThreadName("vnode-commit"); SVnodeTask* pTask; for (;;) { - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); for (;;) { pTask = TD_DLIST_HEAD(&(vnodeMgr.queue)); if (pTask == NULL) { if (vnodeMgr.stop) { - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); return NULL; } else { - pthread_cond_wait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); + taosThreadCondWait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); } } else { TD_DLIST_POP(&(vnodeMgr.queue), pTask); @@ -123,10 +104,10 @@ static void* loop(void* arg) { } } - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); - free(pTask); + taosMemoryFree(pTask); } return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e8bc6873ab0a34ed56d1455857be81c26a2ade54..1db17f37cbea660f83f0c2a549e5d9dbfe80b1ea 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -14,28 +14,25 @@ */ #include "vnodeQuery.h" +#include "executor.h" #include "vnd.h" static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg); int vnodeQueryOpen(SVnode *pVnode) { - return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, - (putReqToQueryQFp)vnodePutReqToVQueryQ, (sendReqToDnodeFp)vnodeSendReqToDnode); + return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, &pVnode->msgCb); } -void vnodeQueryClose(SVnode *pVnode) { - qWorkerDestroy((void **)&pVnode->pQuery); -} +void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in query queue is processing"); - SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta}; + SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config}; switch (pMsg->msgType) { - case TDMT_VND_QUERY: { + case TDMT_VND_QUERY: return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); - } case TDMT_VND_QUERY_CONTINUE: return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg); default: @@ -68,6 +65,10 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { return vnodeGetTableMeta(pVnode, pMsg); case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); + case TDMT_VND_TASK_EXEC: + return tqProcessTaskExec(pVnode->pTq, pMsg); + case TDMT_VND_STREAM_TRIGGER: + return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen); case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: @@ -88,11 +89,24 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { SSchema *pTagSchema; SRpcMsg rpcMsg; int msgLen = 0; - int32_t code = TSDB_CODE_VND_APP_ERROR; + int32_t code = 0; + char tableFName[TSDB_TABLE_FNAME_LEN]; + int32_t rspLen = 0; + void *pRsp = NULL; STableInfoReq infoReq = {0}; if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + metaRsp.dbId = pVnode->config.dbId; + memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); + strcpy(metaRsp.tbName, infoReq.tbName); + + sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); + code = vnodeValidateTableHash(&pVnode->config, tableFName); + if (code) { goto _exit; } @@ -126,15 +140,12 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { pTagSchema = NULL; } - metaRsp.pSchemas = calloc(nCols + nTagCols, sizeof(SSchema)); + metaRsp.pSchemas = taosMemoryCalloc(nCols + nTagCols, sizeof(SSchema)); if (metaRsp.pSchemas == NULL) { code = TSDB_CODE_VND_OUT_OF_MEMORY; goto _exit; } - metaRsp.dbId = pVnode->config.dbId; - memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); - strcpy(metaRsp.tbName, infoReq.tbName); if (pTbCfg->type == META_CHILD_TABLE) { strcpy(metaRsp.stbName, pStbCfg->name); metaRsp.suid = pTbCfg->ctbCfg.suid; @@ -153,38 +164,36 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols); } - int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); +_exit: + + rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); if (rspLen < 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; } - void *pRsp = rpcMallocCont(rspLen); + pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); - code = 0; - -_exit: - tFreeSTableMetaRsp(&metaRsp); if (pSW != NULL) { - tfree(pSW->pSchema); - tfree(pSW); + taosMemoryFreeClear(pSW->pSchema); + taosMemoryFreeClear(pSW); } if (pTbCfg) { - tfree(pTbCfg->name); + taosMemoryFreeClear(pTbCfg->name); if (pTbCfg->type == META_SUPER_TABLE) { - free(pTbCfg->stbCfg.pTagSchema); + taosMemoryFree(pTbCfg->stbCfg.pTagSchema); } else if (pTbCfg->type == META_SUPER_TABLE) { kvRowFree(pTbCfg->ctbCfg.pTag); } - tfree(pTbCfg); + taosMemoryFreeClear(pTbCfg); } rpcMsg.handle = pMsg->handle; @@ -195,12 +204,12 @@ _exit: rpcSendResponse(&rpcMsg); - return code; + return TSDB_CODE_SUCCESS; } static void freeItemHelper(void *pItem) { char *p = *(char **)pItem; - free(p); + taosMemoryFree(p); } /** @@ -220,7 +229,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { taosArrayPush(pArray, &name); totalLen += strlen(name); } else { - tfree(name); + taosMemoryFreeClear(name); } numOfTables++; @@ -250,7 +259,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); - // free(n); + // taosMemoryFree(n); } pFetchRsp->numOfRows = htonl(numOfTables); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 8285020e14a336c08d51287b544d2714e0029b18..e0be9ed89a239cc29fa6f741af785fc4d8266d25 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -13,21 +13,22 @@ * along with this program. If not, see . */ -#include "tq.h" #include "vnd.h" -int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { - SRpcMsg *pMsg; +void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { + SNodeMsg *pMsg; + SRpcMsg *pRpc; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); + pMsg = *(SNodeMsg **)taosArrayGet(pMsgs, i); + pRpc = &pMsg->rpcMsg; // set request version - void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pRpc->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedI64(&pBuf, ver); - if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { + if (walWrite(pVnode->pWal, ver, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) { // TODO: handle error /*ASSERT(false);*/ vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr()); @@ -38,7 +39,8 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { // TODO: Integrate RAFT module here - return 0; + // No results are returned because error handling is difficult + // return 0; } int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { @@ -57,7 +59,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // todo: change the interface here int64_t ver; taosDecodeFixedI64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); - if (tqPushMsg(pVnode->pTq, ptr, pMsg->msgType, ver) < 0) { + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) { // TODO: handle error } @@ -70,42 +72,70 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // TODO: maybe need to clear the request struct - free(vCreateTbReq.stbCfg.pSchema); - free(vCreateTbReq.stbCfg.pTagSchema); - free(vCreateTbReq.name); + taosMemoryFree(vCreateTbReq.stbCfg.pSchema); + taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); + taosMemoryFree(vCreateTbReq.name); break; } case TDMT_VND_CREATE_TABLE: { SVCreateTbBatchReq vCreateTbBatchReq = {0}; + SVCreateTbBatchRsp vCreateTbBatchRsp = {0}; tDeserializeSVCreateTbBatchReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); - for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) { + int reqNum = taosArrayGetSize(vCreateTbBatchReq.pArray); + for (int i = 0; i < reqNum; i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); + + char tableFName[TSDB_TABLE_FNAME_LEN]; + SMsgHead *pHead = (SMsgHead *)pMsg->pCont; + sprintf(tableFName, "%s.%s", pCreateTbReq->dbFName, pCreateTbReq->name); + + int32_t code = vnodeValidateTableHash(&pVnode->config, tableFName); + if (code) { + SVCreateTbRsp rsp; + rsp.code = code; + + taosArrayPush(vCreateTbBatchRsp.rspList, &rsp); + } + if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); } - free(pCreateTbReq->name); + taosMemoryFree(pCreateTbReq->name); if (pCreateTbReq->type == TD_SUPER_TABLE) { - free(pCreateTbReq->stbCfg.pSchema); - free(pCreateTbReq->stbCfg.pTagSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); } else if (pCreateTbReq->type == TD_CHILD_TABLE) { - free(pCreateTbReq->ctbCfg.pTag); + taosMemoryFree(pCreateTbReq->ctbCfg.pTag); } else { - free(pCreateTbReq->ntbCfg.pSchema); + taosMemoryFree(pCreateTbReq->ntbCfg.pSchema); } } vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); taosArrayDestroy(vCreateTbBatchReq.pArray); + if (vCreateTbBatchRsp.rspList) { + int32_t contLen = tSerializeSVCreateTbBatchRsp(NULL, 0, &vCreateTbBatchRsp); + void *msg = rpcMallocCont(contLen); + tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp); + taosArrayDestroy(vCreateTbBatchRsp.rspList); + + *pRsp = taosMemoryCalloc(1, sizeof(SRpcMsg)); + (*pRsp)->msgType = TDMT_VND_CREATE_TABLE_RSP; + (*pRsp)->pCont = msg; + (*pRsp)->contLen = contLen; + (*pRsp)->handle = pMsg->handle; + (*pRsp)->ahandle = pMsg->ahandle; + } break; } case TDMT_VND_ALTER_STB: { SVCreateTbReq vAlterTbReq = {0}; vTrace("vgId:%d, process alter stb req", pVnode->vgId); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); - free(vAlterTbReq.stbCfg.pSchema); - free(vAlterTbReq.stbCfg.pTagSchema); - free(vAlterTbReq.name); + taosMemoryFree(vAlterTbReq.stbCfg.pSchema); + taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); + taosMemoryFree(vAlterTbReq.name); break; } case TDMT_VND_DROP_STB: @@ -138,6 +168,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } } break; case TDMT_VND_CREATE_SMA: { // timeRangeSMA +#if 0 SSmaCfg vCreateSmaReq = {0}; if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -159,26 +190,37 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // } tdDestroyTSma(&vCreateSmaReq.tSma); // TODO: return directly or go on follow steps? +#endif } break; case TDMT_VND_CANCEL_SMA: { // timeRangeSMA } break; case TDMT_VND_DROP_SMA: { // timeRangeSMA +#if 0 SVDropTSmaReq vDropSmaReq = {0}; if (tDeserializeSVDropTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vDropSmaReq) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexName) < 0) { - // TODO: handle error - return -1; - } // TODO: send msg to stream computing to drop tSma // if ((send msg to stream computing) < 0) { // tdDestroyTSma(&vCreateSmaReq); // return -1; // } + // + + if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + if(tsdbDropTSmaData(pVnode->pTsdb, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + // TODO: return directly or go on follow steps? +#endif } break; default: ASSERT(0); diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp index 4f1518525476127241401741c371f0a8071dc6d9..627dbc6f18122e39cde2e243fd564ad622324cd2 100644 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ b/source/dnode/vnode/test/tqMetaTest.cpp @@ -12,7 +12,7 @@ struct Foo { int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { Foo* foo = (Foo*)pObj; if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { - *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); + *ppHead = (STqSerializedHead*)taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); } *(int32_t*)(*ppHead)->content = foo->a; @@ -21,14 +21,14 @@ int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { if (*ppObj == NULL) { - *ppObj = realloc(*ppObj, sizeof(int32_t)); + *ppObj = taosMemoryRealloc(*ppObj, sizeof(int32_t)); } Foo* pFoo = *(Foo**)ppObj; pFoo->a = *(int32_t*)pHead->content; return NULL; } -void FooDeleter(void* pObj) { free(pObj); } +void FooDeleter(void* pObj) { taosMemoryFree(pObj); } class TqMetaUpdateAppendTest : public ::testing::Test { protected: @@ -58,7 +58,7 @@ TEST_F(TqMetaUpdateAppendTest, copyPutTest) { } TEST_F(TqMetaUpdateAppendTest, persistTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 2; tqHandleMovePut(pMeta, 1, pFoo); Foo* pBar = (Foo*)tqHandleGet(pMeta, 1); @@ -82,7 +82,7 @@ TEST_F(TqMetaUpdateAppendTest, persistTest) { } TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -91,7 +91,7 @@ TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { } TEST_F(TqMetaUpdateAppendTest, abortTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -104,7 +104,7 @@ TEST_F(TqMetaUpdateAppendTest, abortTest) { } TEST_F(TqMetaUpdateAppendTest, deleteTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -135,12 +135,12 @@ TEST_F(TqMetaUpdateAppendTest, deleteTest) { } TEST_F(TqMetaUpdateAppendTest, intxnPersist) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); tqHandleCommit(pMeta, 1); - Foo* pBar = (Foo*)malloc(sizeof(Foo)); + Foo* pBar = (Foo*)taosMemoryMalloc(sizeof(Foo)); pBar->a = 4; tqHandleMovePut(pMeta, 1, pBar); diff --git a/source/dnode/vnode/test/tqSerializerTest.cpp b/source/dnode/vnode/test/tqSerializerTest.cpp deleted file mode 100644 index 0d76322c17d79ebca9c0164e1762844c5021c4fe..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/test/tqSerializerTest.cpp +++ /dev/null @@ -1,13 +0,0 @@ -#include -#include -#include -#include - -#include "tq.h" - -using namespace std; - -TEST(TqSerializerTest, basicTest) { - TqGroupHandle* gHandle = (TqGroupHandle*)malloc(sizeof(TqGroupHandle)); - -} diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 5a87c180b67ec371a30c969bd0931fdc166b6a44..86db3af4dc7e92bf37d3535d67e4fc51fa73fb5d 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -14,13 +14,13 @@ */ #include +#include #include #include #include #include #include -#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -33,6 +33,62 @@ int main(int argc, char **argv) { return RUN_ALL_TESTS(); } +TEST(testCase, unionEncodeDecodeTest) { + typedef struct { + union { + uint8_t info; + struct { + uint8_t rollup : 1; // 1 means rollup sma + uint8_t type : 7; + }; + }; + col_id_t nBSmaCols; + col_id_t *pBSmaCols; + } SUnionTest; + + SUnionTest sut = {0}; + sut.rollup = 1; + sut.type = 1; + + sut.nBSmaCols = 2; + sut.pBSmaCols = (col_id_t *)taosMemoryMalloc(sut.nBSmaCols * sizeof(col_id_t)); + for (col_id_t i = 0; i < sut.nBSmaCols; ++i) { + sut.pBSmaCols[i] = i + 100; + } + + void *buf = taosMemoryMalloc(1024); + void *pBuf = buf; + int32_t tlen = 0; + tlen += taosEncodeFixedU8(&buf, sut.info); + tlen += taosEncodeFixedI16(&buf, sut.nBSmaCols); + for (col_id_t i = 0; i < sut.nBSmaCols; ++i) { + tlen += taosEncodeFixedI16(&buf, sut.pBSmaCols[i]); + } + + SUnionTest dut = {0}; + pBuf = taosDecodeFixedU8(pBuf, &dut.info); + pBuf = taosDecodeFixedI16(pBuf, &dut.nBSmaCols); + if (dut.nBSmaCols > 0) { + dut.pBSmaCols = (col_id_t *)taosMemoryMalloc(dut.nBSmaCols * sizeof(col_id_t)); + for (col_id_t i = 0; i < dut.nBSmaCols; ++i) { + pBuf = taosDecodeFixedI16(pBuf, dut.pBSmaCols + i); + } + } else { + dut.pBSmaCols = NULL; + } + + printf("sut.rollup=%" PRIu8 ", type=%" PRIu8 ", info=%" PRIu8 "\n", sut.rollup, sut.type, sut.info); + printf("dut.rollup=%" PRIu8 ", type=%" PRIu8 ", info=%" PRIu8 "\n", dut.rollup, dut.type, dut.info); + + ASSERT_EQ(sut.rollup, dut.rollup); + ASSERT_EQ(sut.type, dut.type); + ASSERT_EQ(sut.nBSmaCols, dut.nBSmaCols); + for (col_id_t i = 0; i < sut.nBSmaCols; ++i) { + ASSERT_EQ(*(col_id_t *)(sut.pBSmaCols + i), sut.pBSmaCols[i]); + ASSERT_EQ(*(col_id_t *)(sut.pBSmaCols + i), dut.pBSmaCols[i]); + } +} +#if 1 TEST(testCase, tSma_Meta_Encode_Decode_Test) { // encode STSma tSma = {0}; @@ -42,14 +98,14 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { tSma.slidingUnit = TIME_UNIT_HOUR; tSma.sliding = 0; tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN); - tstrncpy(tSma.timezone, "Asia/Shanghai", TD_TIMEZONE_LEN); + tSma.timezoneInt = 8; tSma.indexUid = 2345678910; tSma.tableUid = 1234567890; STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma}; uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper); - void *buf = calloc(1, bufLen); + void *buf = taosMemoryCalloc(1, bufLen); ASSERT_NE(buf, nullptr); STSmaWrapper *pSW = (STSmaWrapper *)buf; @@ -59,7 +115,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { // decode STSmaWrapper dstTSmaWrapper = {0}; - void * result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper); + void *result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper); ASSERT_NE(result, nullptr); ASSERT_EQ(tSmaWrapper.number, dstTSmaWrapper.number); @@ -72,7 +128,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { ASSERT_EQ(pSma->intervalUnit, qSma->intervalUnit); ASSERT_EQ(pSma->slidingUnit, qSma->slidingUnit); ASSERT_STRCASEEQ(pSma->indexName, qSma->indexName); - ASSERT_STRCASEEQ(pSma->timezone, qSma->timezone); + ASSERT_EQ(pSma->timezoneInt, qSma->timezoneInt); ASSERT_EQ(pSma->indexUid, qSma->indexUid); ASSERT_EQ(pSma->tableUid, qSma->tableUid); ASSERT_EQ(pSma->interval, qSma->interval); @@ -84,19 +140,20 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { } // resource release - tfree(pSW); + taosMemoryFreeClear(pSW); tdDestroyTSma(&tSma); tdDestroyTSmaWrapper(&dstTSmaWrapper); } +#endif #if 1 TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { - const char * smaIndexName1 = "sma_index_test_1"; - const char * smaIndexName2 = "sma_index_test_2"; - const char * timezone = "Asia/Shanghai"; - const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; - const char * tagsFilter = "I'm tags filter"; - const char * smaTestDir = "./smaTest"; + const char *smaIndexName1 = "sma_index_test_1"; + const char *smaIndexName2 = "sma_index_test_2"; + int8_t timezone = 8; + const char *expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; + const char *tagsFilter = "I'm tags filter"; + const char *smaTestDir = "./smaTest"; const tb_uid_t tbUid = 1234567890; const int64_t indexUid1 = 2000000001; const int64_t indexUid2 = 2000000002; @@ -110,21 +167,21 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { tSma.sliding = 0; tSma.indexUid = indexUid1; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); - tstrncpy(tSma.timezone, timezone, TD_TIMEZONE_LEN); + tSma.timezoneInt = 8; tSma.tableUid = tbUid; tSma.exprLen = strlen(expr); - tSma.expr = (char *)calloc(1, tSma.exprLen + 1); + tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1); ASSERT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); - tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); + tSma.tagsFilter = (char *)taosMemoryCalloc(tSma.tagsFilterLen + 1, 1); ASSERT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); - SMeta * pMeta = NULL; - STSma * pSmaCfg = &tSma; + SMeta *pMeta = NULL; + STSma *pSmaCfg = &tSma; const SMetaCfg *pMetaCfg = &defaultMetaOptions; taosRemoveDir(smaTestDir); @@ -150,24 +207,24 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid1); assert(qSmaCfg != NULL); printf("name1 = %s\n", qSmaCfg->indexName); - printf("timezone1 = %s\n", qSmaCfg->timezone); + printf("timezone1 = %" PRIi8 "\n", qSmaCfg->timezoneInt); printf("expr1 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); ASSERT_EQ(qSmaCfg->tableUid, tSma.tableUid); tdDestroyTSma(qSmaCfg); - tfree(qSmaCfg); + taosMemoryFreeClear(qSmaCfg); qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2); assert(qSmaCfg != NULL); printf("name2 = %s\n", qSmaCfg->indexName); - printf("timezone2 = %s\n", qSmaCfg->timezone); + printf("timezone2 = %" PRIi8 "\n", qSmaCfg->timezoneInt); printf("expr2 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); ASSERT_EQ(qSmaCfg->interval, tSma.interval); tdDestroyTSma(qSmaCfg); - tfree(qSmaCfg); + taosMemoryFreeClear(qSmaCfg); // get index name by table uid SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid); @@ -189,20 +246,20 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { assert(pSW != NULL); ASSERT_EQ(pSW->number, nCntTSma); ASSERT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); - ASSERT_STRCASEEQ(pSW->tSma->timezone, timezone); + ASSERT_EQ(pSW->tSma->timezoneInt, timezone); ASSERT_STRCASEEQ(pSW->tSma->expr, expr); ASSERT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter); ASSERT_EQ(pSW->tSma->indexUid, indexUid1); ASSERT_EQ(pSW->tSma->tableUid, tbUid); ASSERT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); - ASSERT_STRCASEEQ((pSW->tSma + 1)->timezone, timezone); + ASSERT_EQ((pSW->tSma + 1)->timezoneInt, timezone); ASSERT_STRCASEEQ((pSW->tSma + 1)->expr, expr); ASSERT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter); ASSERT_EQ((pSW->tSma + 1)->indexUid, indexUid2); ASSERT_EQ((pSW->tSma + 1)->tableUid, tbUid); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); // get all sma table uids SArray *pUids = metaGetSmaTbUids(pMeta, false); @@ -215,8 +272,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { taosArrayDestroy(pUids); // resource release - metaRemoveSmaFromDb(pMeta, smaIndexName1); - metaRemoveSmaFromDb(pMeta, smaIndexName2); + metaRemoveSmaFromDb(pMeta, indexUid1); + metaRemoveSmaFromDb(pMeta, indexUid2); tdDestroyTSma(&tSma); metaClose(pMeta); @@ -226,11 +283,11 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { #if 1 TEST(testCase, tSma_Data_Insert_Query_Test) { // step 1: prepare meta - const char * smaIndexName1 = "sma_index_test_1"; - const char * timezone = "Asia/Shanghai"; - const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; - const char * tagsFilter = "where tags.location='Beijing' and tags.district='ChaoYang'"; - const char * smaTestDir = "./smaTest"; + const char *smaIndexName1 = "sma_index_test_1"; + const int8_t timezone = 8; + const char *expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; + const char *tagsFilter = "where tags.location='Beijing' and tags.district='ChaoYang'"; + const char *smaTestDir = "./smaTest"; const tb_uid_t tbUid = 1234567890; const int64_t indexUid1 = 2000000001; const int64_t interval1 = 1; @@ -245,24 +302,24 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { tSma.intervalUnit = TIME_UNIT_DAY; tSma.interval = 1; tSma.slidingUnit = TIME_UNIT_HOUR; - tSma.sliding = 0; + tSma.sliding = 1; // sliding = interval when it's convert window tSma.indexUid = indexUid1; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); - tstrncpy(tSma.timezone, timezone, TD_TIMEZONE_LEN); + tSma.timezoneInt = timezone; tSma.tableUid = tbUid; tSma.exprLen = strlen(expr); - tSma.expr = (char *)calloc(1, tSma.exprLen + 1); + tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1); ASSERT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); - tSma.tagsFilter = (char *)calloc(1, tSma.tagsFilterLen + 1); + tSma.tagsFilter = (char *)taosMemoryCalloc(1, tSma.tagsFilterLen + 1); ASSERT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); - SMeta * pMeta = NULL; - STSma * pSmaCfg = &tSma; + SMeta *pMeta = NULL; + STSma *pSmaCfg = &tSma; const SMetaCfg *pMetaCfg = &defaultMetaOptions; taosRemoveDir(smaTestDir); @@ -274,8 +331,8 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { // step 2: insert data STSmaDataWrapper *pSmaData = NULL; - STsdb * pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); - STsdbCfg * pCfg = &pTsdb->config; + STsdb *pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); + STsdbCfg *pCfg = &pTsdb->config; pTsdb->pMeta = pMeta; pTsdb->vgId = 2; @@ -302,21 +359,57 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { break; } - SDiskCfg pDisks = {.level = 0, .primary = 1}; + SDiskCfg pDisks = {0}; + pDisks.level = 0; + pDisks.primary = 1; strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN); int32_t numOfDisks = 1; pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); ASSERT_NE(pTsdb->pTfs, nullptr); - char *msg = (char *)calloc(1, 100); - ASSERT_NE(msg, nullptr); - ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); + // generate SSubmitReq msg and update expired window + int16_t schemaVer = 0; + uint32_t mockRowLen = sizeof(STSRow); + uint32_t mockRowNum = 2; + uint32_t mockBlkNum = 2; + uint32_t msgLen = sizeof(SSubmitReq) + mockBlkNum * sizeof(SSubmitBlk) + mockBlkNum * mockRowNum * mockRowLen; + + SSubmitReq *pMsg = (SSubmitReq *)taosMemoryCalloc(1, msgLen); + ASSERT_NE(pMsg, nullptr); + pMsg->version = htobe64(schemaVer); + pMsg->numOfBlocks = htonl(mockBlkNum); + pMsg->length = htonl(msgLen); + + SSubmitBlk *pBlk = NULL; + STSRow *pRow = NULL; + TSKEY now = taosGetTimestamp(pTsdb->config.precision); + + for (uint32_t b = 0; b < mockBlkNum; ++b) { + pBlk = (SSubmitBlk *)POINTER_SHIFT(pMsg, sizeof(SSubmitReq) + b * (sizeof(SSubmitBlk) + mockRowNum * mockRowLen)); + pBlk->uid = htobe64(tbUid); + pBlk->suid = htobe64(tbUid); + pBlk->sversion = htonl(schemaVer); + pBlk->padding = htonl(0); + pBlk->schemaLen = htonl(0); + pBlk->numOfRows = htons(mockRowNum); + pBlk->dataLen = htonl(mockRowNum * mockRowLen); + for (uint32_t r = 0; r < mockRowNum; ++r) { + pRow = (STSRow *)POINTER_SHIFT(pBlk, sizeof(SSubmitBlk) + r * mockRowLen); + pRow->len = mockRowLen; + pRow->ts = now + b * 1000 + r * 1000; + pRow->sver = schemaVer; + } + } + + ASSERT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS); + + ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, (const char *)pMsg), 0); // init int32_t allocCnt = 0; int32_t allocStep = 16384; int32_t buffer = 1024; - void * buf = NULL; + void *buf = NULL; ASSERT_EQ(tsdbMakeRoom(&buf, allocStep), 0); int32_t bufSize = taosTSizeof(buf); int32_t numOfTables = 10; @@ -384,7 +477,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt); // release data - tfree(msg); + taosMemoryFreeClear(pMsg); taosTZfree(buf); // release meta tdDestroyTSma(&tSma); @@ -392,6 +485,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { tsdbClose(pTsdb); metaClose(pMeta); } + #endif #pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index ea8195bfd11081dd506ee61302a8141c99038b02..b448a43dcb332804e4c91de89e6fb1f2c8549770 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(scheduler) add_subdirectory(cache) add_subdirectory(catalog) add_subdirectory(executor) +add_subdirectory(stream) add_subdirectory(planner) add_subdirectory(function) add_subdirectory(qcom) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c4f1a117fe58e146efcc393287700db8c9d131e4..9bd5c1c016e9f0483b36c83131c482087d84f4e3 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -30,6 +30,7 @@ extern "C" { #define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000 #define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10 +#define CTG_DEFAULT_MAX_RETRY_TIMES 3 #define CTG_RENT_SLOT_SECOND 1.5 @@ -57,10 +58,10 @@ enum { }; typedef struct SCtgDebug { - bool lockDebug; - bool cacheDebug; - bool apiDebug; - bool metaDebug; + bool lockEnable; + bool cacheEnable; + bool apiEnable; + bool metaEnable; uint32_t showCachePeriodSec; } SCtgDebug; @@ -159,8 +160,10 @@ typedef struct SCtgRemoveTblMsg { typedef struct SCtgMetaAction { - int32_t act; - void *data; + int32_t act; + void *data; + bool syncReq; + uint64_t seqId; } SCtgMetaAction; typedef struct SCtgQNode { @@ -168,15 +171,22 @@ typedef struct SCtgQNode { struct SCtgQNode *next; } SCtgQNode; -typedef struct SCatalogMgmt { - bool exit; - SRWLatch lock; +typedef struct SCtgQueue { SRWLatch qlock; + uint64_t seqId; + uint64_t seqDone; SCtgQNode *head; SCtgQNode *tail; - tsem_t sem; + tsem_t reqSem; + tsem_t rspSem; uint64_t qRemainNum; - pthread_t updateThread; +} SCtgQueue; + +typedef struct SCatalogMgmt { + bool exit; + SRWLatch lock; + SCtgQueue queue; + TdThread updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; @@ -191,8 +201,8 @@ typedef struct SCtgAction { ctgActFunc func; } SCtgAction; -#define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.qRemainNum, 1) -#define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.qRemainNum, 1) +#define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) #define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) #define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) @@ -232,9 +242,9 @@ typedef struct SCtgAction { #define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__) #define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__) -#define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) -#define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheDebug) { qDebug(__VA_ARGS__); } } while (0) -#define CTG_API_DEBUG(...) do { if (gCTGDebug.apiDebug) { qDebug(__VA_ARGS__); } } while (0) +#define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockEnable) { qDebug(__VA_ARGS__); } } while (0) +#define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheEnable) { qDebug(__VA_ARGS__); } } while (0) +#define CTG_API_DEBUG(...) do { if (gCTGDebug.apiEnable) { qDebug(__VA_ARGS__); } } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 @@ -278,7 +288,7 @@ typedef struct SCtgAction { #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) -#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8(&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) +#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 08e0ac55d7ecbf1267d9962242c5c8acf59c2539..6eab0d280ba326d979dadafeeb6e03fb0e78a233 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -55,25 +55,25 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ int32_t ctgDbgEnableDebug(char *option) { if (0 == strcasecmp(option, "lock")) { - gCTGDebug.lockDebug = true; + gCTGDebug.lockEnable = true; qDebug("lock debug enabled"); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "cache")) { - gCTGDebug.cacheDebug = true; + gCTGDebug.cacheEnable = true; qDebug("cache debug enabled"); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "api")) { - gCTGDebug.apiDebug = true; + gCTGDebug.apiEnable = true; qDebug("api debug enabled"); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "meta")) { - gCTGDebug.metaDebug = true; + gCTGDebug.metaEnable = true; qDebug("api debug enabled"); return TSDB_CODE_SUCCESS; } @@ -155,7 +155,7 @@ int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { } void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { - if (!gCTGDebug.metaDebug) { + if (!gCTGDebug.metaEnable) { return; } @@ -177,7 +177,7 @@ void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { } void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { - if (NULL == dbHash || !gCTGDebug.cacheDebug) { + if (NULL == dbHash || !gCTGDebug.cacheEnable) { return; } @@ -190,7 +190,7 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbCache = (SCtgDBCache *)pIter; - taosHashGetKey((void **)&dbFName, &len); + dbFName = taosHashGetKey(pIter, &len); int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; @@ -217,7 +217,7 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { void ctgDbgShowClusterCache(SCatalog* pCtg) { - if (!gCTGDebug.cacheDebug || NULL == pCtg) { + if (!gCTGDebug.cacheEnable || NULL == pCtg) { return; } @@ -229,93 +229,187 @@ void ctgDbgShowClusterCache(SCatalog* pCtg) { } +void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { + if (NULL == mgmt->slots) { + return; + } + + for (int32_t i = 0; i < mgmt->slotNum; ++i) { + SCtgRentSlot *slot = &mgmt->slots[i]; + if (slot->meta) { + taosArrayDestroy(slot->meta); + slot->meta = NULL; + } + } + + taosMemoryFreeClear(mgmt->slots); +} + + +void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { + CTG_LOCK(CTG_WRITE, &cache->stbLock); + if (cache->stbCache) { + taosHashCleanup(cache->stbCache); + cache->stbCache = NULL; + } + CTG_UNLOCK(CTG_WRITE, &cache->stbLock); + + CTG_LOCK(CTG_WRITE, &cache->metaLock); + if (cache->metaCache) { + taosHashCleanup(cache->metaCache); + cache->metaCache = NULL; + } + CTG_UNLOCK(CTG_WRITE, &cache->metaLock); +} + +void ctgFreeVgInfo(SDBVgInfo *vgInfo) { + if (NULL == vgInfo) { + return; + } + + if (vgInfo->vgHash) { + taosHashCleanup(vgInfo->vgHash); + vgInfo->vgHash = NULL; + } + + taosMemoryFreeClear(vgInfo); +} + +void ctgFreeDbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache) { + return; + } + + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + ctgFreeVgInfo (dbCache->vgInfo); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + + ctgFreeTableMetaCache(&dbCache->tbCache); +} + + +void ctgFreeHandle(SCatalog* pCtg) { + ctgFreeMetaRent(&pCtg->dbRent); + ctgFreeMetaRent(&pCtg->stbRent); + + if (pCtg->dbCache) { + void *pIter = taosHashIterate(pCtg->dbCache, NULL); + while (pIter) { + SCtgDBCache *dbCache = pIter; + + atomic_store_8(&dbCache->deleted, 1); + + ctgFreeDbCache(dbCache); + + pIter = taosHashIterate(pCtg->dbCache, pIter); + } + + taosHashCleanup(pCtg->dbCache); + } + + taosMemoryFree(pCtg); +} + + + +void ctgWaitAction(SCtgMetaAction *action) { + while (true) { + tsem_wait(&gCtgMgmt.queue.rspSem); + + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + tsem_post(&gCtgMgmt.queue.rspSem); + break; + } + + if (gCtgMgmt.queue.seqDone >= action->seqId) { + break; + } + + tsem_post(&gCtgMgmt.queue.rspSem); + sched_yield(); + } +} void ctgPopAction(SCtgMetaAction **action) { - SCtgQNode *orig = gCtgMgmt.head; + SCtgQNode *orig = gCtgMgmt.queue.head; - SCtgQNode *node = gCtgMgmt.head->next; - gCtgMgmt.head = gCtgMgmt.head->next; + SCtgQNode *node = gCtgMgmt.queue.head->next; + gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; CTG_QUEUE_SUB(); - tfree(orig); + taosMemoryFreeClear(orig); *action = &node->action; } -int32_t ctgPushAction(SCtgMetaAction *action) { - SCtgQNode *node = calloc(1, sizeof(SCtgQNode)); +int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { + SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); CTG_RET(TSDB_CODE_CTG_MEM_ERROR); } + + action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1); node->action = *action; - CTG_LOCK(CTG_WRITE, &gCtgMgmt.qlock); - gCtgMgmt.tail->next = node; - gCtgMgmt.tail = node; - CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.qlock); + CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); + gCtgMgmt.queue.tail->next = node; + gCtgMgmt.queue.tail = node; + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); CTG_QUEUE_ADD(); CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); - tsem_post(&gCtgMgmt.sem); - - return TSDB_CODE_SUCCESS; -} + tsem_post(&gCtgMgmt.queue.reqSem); + ctgDebug("action [%s] added into queue", gCtgAction[action->act].name); -void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { - if (NULL == mgmt->slots) { - return; + if (action->syncReq) { + ctgWaitAction(action); } - for (int32_t i = 0; i < mgmt->slotNum; ++i) { - SCtgRentSlot *slot = &mgmt->slots[i]; - if (slot->meta) { - taosArrayDestroy(slot->meta); - slot->meta = NULL; - } - } - - tfree(mgmt->slots); + return TSDB_CODE_SUCCESS; } int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { int32_t code = 0; SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; - SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); + SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_INF_DBNAME(p + 1)) { + dbFName = p + 1; + } + msg->pCtg = pCtg; strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->dbId = dbId; action.data = msg; - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); return TSDB_CODE_SUCCESS; _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } -int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid) { +int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) { int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; - SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; + SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -329,24 +423,22 @@ int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId action.data = msg; - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); return TSDB_CODE_SUCCESS; _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } -int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName) { +int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL}; - SCtgRemoveTblMsg *msg = malloc(sizeof(SCtgRemoveTblMsg)); + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; + SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -359,81 +451,77 @@ int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId action.data = msg; - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); return TSDB_CODE_SUCCESS; _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } - -void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - taosHashCleanup(cache->stbCache); - cache->stbCache = NULL; +int32_t ctgPushUpdateVgMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; + SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + ctgFreeVgInfo(dbInfo); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - CTG_LOCK(CTG_WRITE, &cache->metaLock); - if (cache->metaCache) { - taosHashCleanup(cache->metaCache); - cache->metaCache = NULL; + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_INF_DBNAME(p + 1)) { + dbFName = p + 1; } - CTG_UNLOCK(CTG_WRITE, &cache->metaLock); -} -void ctgFreeVgInfo(SDBVgInfo *vgInfo) { - if (NULL == vgInfo) { - return; - } + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->pCtg = pCtg; + msg->dbId = dbId; + msg->dbInfo = dbInfo; - if (vgInfo->vgHash) { - taosHashCleanup(vgInfo->vgHash); - vgInfo->vgHash = NULL; - } - - tfree(vgInfo); -} + action.data = msg; -void ctgFreeDbCache(SCtgDBCache *dbCache) { - if (NULL == dbCache) { - return; - } + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo (dbCache->vgInfo); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + return TSDB_CODE_SUCCESS; - ctgFreeTableMetaCache(&dbCache->tbCache); +_return: + + ctgFreeVgInfo(dbInfo); + taosMemoryFreeClear(action.data); + CTG_RET(code); } +int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; + SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } -void ctgFreeHandle(SCatalog* pCtg) { - ctgFreeMetaRent(&pCtg->dbRent); - ctgFreeMetaRent(&pCtg->stbRent); - - if (pCtg->dbCache) { - void *pIter = taosHashIterate(pCtg->dbCache, NULL); - while (pIter) { - SCtgDBCache *dbCache = pIter; + char *p = strchr(output->dbFName, '.'); + if (p && CTG_IS_INF_DBNAME(p + 1)) { + memmove(output->dbFName, p + 1, strlen(p + 1)); + } - atomic_store_8(&dbCache->deleted, 1); + msg->pCtg = pCtg; + msg->output = output; - ctgFreeDbCache(dbCache); - - pIter = taosHashIterate(pCtg->dbCache, pIter); - } + action.data = msg; - taosHashCleanup(pCtg->dbCache); - } + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; - free(pCtg); +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); } @@ -489,6 +577,11 @@ void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_INF_DBNAME(p + 1)) { + dbFName = p + 1; + } + SCtgDBCache *dbCache = NULL; if (acquire) { dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); @@ -725,7 +818,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; } @@ -733,13 +826,13 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = CTG_META_SIZE(*stbMeta); - *pTableMeta = realloc(*pTableMeta, metaSize); + *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); @@ -758,18 +851,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType, int32_t flag) { +int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, tbName:%s", pTableName->tname); + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); return TSDB_CODE_SUCCESS; } - - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_FLAG_IS_INF_DB(flag)) { - strcpy(dbFName, pTableName->dbname); - } else { - tNameGetFullDbName(pTableName, dbFName); - } SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); @@ -778,11 +864,11 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); if (NULL == pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); ctgReleaseDBCache(pCtg, dbCache); return TSDB_CODE_SUCCESS; @@ -796,7 +882,7 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, pTableName->tname, *tbType); + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); return TSDB_CODE_SUCCESS; } @@ -854,7 +940,7 @@ int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMg return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromVnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -904,6 +990,32 @@ int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMg return TSDB_CODE_SUCCESS; } +int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + int32_t code = 0; + int32_t retryNum = 0; + + while (retryNum < CTG_DEFAULT_MAX_RETRY_TIMES) { + code = ctgGetTableMetaFromVnodeImpl(pCtg, pTrans, pMgmtEps, pTableName, vgroupInfo, output); + if (code) { + if (TSDB_CODE_VND_HASH_MISMATCH == code) { + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + + code = catalogRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName); + if (code != TSDB_CODE_SUCCESS) { + break; + } + + ++retryNum; + continue; + } + } + + break; + } + + CTG_RET(code); +} int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { switch (hashMethod) { @@ -1049,7 +1161,7 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; - mgmt->slots = calloc(1, msgSize); + mgmt->slots = taosMemoryCalloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1193,7 +1305,7 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_ } size_t msize = metaNum * size; - *res = malloc(msize); + *res = taosMemoryMalloc(msize); if (NULL == *res) { qError("malloc %d failed", (int32_t)msize); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -1265,16 +1377,12 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - if (CTG_IS_INF_DBNAME(dbFName)) { - return TSDB_CODE_SUCCESS; - } - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); @@ -1319,8 +1427,6 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { ctgFreeDbCache(dbCache); - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); @@ -1381,9 +1487,14 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { int32_t code = 0; SDBVgInfo* dbInfo = *pDbInfo; + + if (NULL == dbInfo->vgHash) { + return TSDB_CODE_SUCCESS; + } - if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion); + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", + dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1520,7 +1631,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui } int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { - *dst = malloc(sizeof(SDBVgInfo)); + *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); if (NULL == *dst) { qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1532,7 +1643,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == (*dst)->vgHash) { qError("taosHashInit %d failed", (int32_t)hashSize); - tfree(*dst); + taosMemoryFreeClear(*dst); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1545,7 +1656,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); taosHashCancelIterate(src->vgHash, pIter); taosHashCleanup((*dst)->vgHash); - tfree(*dst); + taosMemoryFreeClear(*dst); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1558,13 +1669,13 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { -int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { +int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { bool inCache = false; int32_t code = 0; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); - if (inCache && !forceUpdate) { + if (inCache) { return TSDB_CODE_SUCCESS; } @@ -1572,13 +1683,7 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); - if (inCache) { - input.dbId = (*dbCache)->dbId; - input.vgVersion = (*dbCache)->vgInfo->vgVersion; - input.numOfTable = (*dbCache)->vgInfo->numOfTable; - } else { - input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - } + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); if (code) { @@ -1591,41 +1696,59 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const } CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); - - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG}; - SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); - ctgFreeVgInfo(DbOut.dbVgroup); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - msg->pCtg = pCtg; - msg->dbId = DbOut.dbId; - msg->dbInfo = DbOut.dbVgroup; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_RET(ctgPushUpdateVgMsgInQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, false)); return TSDB_CODE_SUCCESS; _return: - tfree(*pInfo); - tfree(msg); - + taosMemoryFreeClear(*pInfo); *pInfo = DbOut.dbVgroup; CTG_RET(code); } +int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName) { + bool inCache = false; + int32_t code = 0; + SCtgDBCache* dbCache = NULL; + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); + + SUseDbOutput DbOut = {0}; + SBuildUseDBInput input = {0}; + tstrncpy(input.db, dbFName, tListLen(input.db)); + + if (inCache) { + input.dbId = dbCache->dbId; + + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + input.numOfTable = 0; + + code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); + if (code) { + if (CTG_DB_NOT_EXIST(code) && inCache) { + ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); + ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId); + } + + CTG_ERR_RET(code); + } + + CTG_ERR_RET(ctgPushUpdateVgMsgInQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, true)); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { - *pOutput = malloc(sizeof(STableMetaOutput)); + *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); if (NULL == *pOutput) { qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1635,10 +1758,10 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) if (output->tbMeta) { int32_t metaSize = CTG_META_SIZE(output->tbMeta); - (*pOutput)->tbMeta = malloc(metaSize); + (*pOutput)->tbMeta = taosMemoryMalloc(metaSize); if (NULL == (*pOutput)->tbMeta) { qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); - tfree(*pOutput); + taosMemoryFreeClear(*pOutput); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1650,7 +1773,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) -int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t flag, STableMetaOutput **pOutput) { +int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t flag, STableMetaOutput **pOutput, bool syncReq) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1662,9 +1785,8 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); } - SCtgUpdateTblMsg *msg = NULL; STableMetaOutput moutput = {0}; - STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1692,7 +1814,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { @@ -1708,11 +1830,11 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SET_META_TYPE_NULL(output->metaType); } - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); SET_META_TYPE_CTABLE(output->metaType); } @@ -1721,6 +1843,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_IS_META_NULL(output->metaType)) { ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName)); + catalogRemoveTableMeta(pCtg, pTableName); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1734,29 +1857,14 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); } - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; - msg = malloc(sizeof(SCtgUpdateTblMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - msg->output = output; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, syncReq)); return TSDB_CODE_SUCCESS; _return: - tfree(output->tbMeta); - tfree(output); - tfree(msg); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); CTG_RET(code); } @@ -1788,7 +1896,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons tbType = (*pTableMeta)->tableType; suid = (*pTableMeta)->suid; - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); } if (CTG_FLAG_IS_UNKNOWN_STB(flag)) { @@ -1797,7 +1905,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons while (true) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output, false)); if (CTG_IS_META_TABLE(output->metaType)) { *pTableMeta = output->tbMeta; @@ -1813,7 +1921,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { ctgError("invalid metaType:%d", output->metaType); - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1844,13 +1952,13 @@ _return: } if (TSDB_SUPER_TABLE == tbType) { - ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid); + ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid, false); } else { - ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname); + ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, false); } } - tfree(output); + taosMemoryFreeClear(output); if (*pTableMeta) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); @@ -1871,7 +1979,7 @@ int32_t ctgActUpdateVg(SCtgMetaAction *action) { _return: ctgFreeVgInfo(msg->dbInfo); - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -1896,7 +2004,7 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -1918,11 +2026,6 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - - char *p = strchr(output->dbFName, '.'); - if (p && CTG_IS_INF_DBNAME(p + 1)) { - memmove(output->dbFName, p + 1, strlen(p + 1)); - } CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); if (NULL == dbCache) { @@ -1943,11 +2046,11 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { _return: if (output) { - tfree(output->tbMeta); - tfree(output); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); } - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -1964,24 +2067,19 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { return TSDB_CODE_SUCCESS; } - if (dbCache->dbId != msg->dbId) { + if (msg->dbId && (dbCache->dbId != msg->dbId)) { ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); return TSDB_CODE_SUCCESS; } CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - return TSDB_CODE_SUCCESS; } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); @@ -1995,7 +2093,7 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2028,7 +2126,7 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2042,9 +2140,10 @@ void* ctgUpdateThreadFunc(void* param) { CTG_LOCK(CTG_READ, &gCtgMgmt.lock); while (true) { - tsem_wait(&gCtgMgmt.sem); + tsem_wait(&gCtgMgmt.queue.reqSem); - if (atomic_load_8(&gCtgMgmt.exit)) { + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + tsem_post(&gCtgMgmt.queue.rspSem); break; } @@ -2056,6 +2155,12 @@ void* ctgUpdateThreadFunc(void* param) { (*gCtgAction[action->act].func)(action); + gCtgMgmt.queue.seqDone = action->seqId; + + if (action->syncReq) { + tsem_post(&gCtgMgmt.queue.rspSem); + } + CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); ctgDbgShowClusterCache(pCtg); @@ -2070,19 +2175,95 @@ void* ctgUpdateThreadFunc(void* param) { int32_t ctgStartUpdateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); CTG_ERR_RET(terrno); } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); return TSDB_CODE_SUCCESS; } +int32_t ctgGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { + STableMeta *tbMeta = NULL; + int32_t code = 0; + SVgroupInfo vgroupInfo = {0}; + SCtgDBCache* dbCache = NULL; + SArray *vgList = NULL; + SDBVgInfo *vgInfo = NULL; + + *pVgList = NULL; + + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); + + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); + + SHashObj *vgHash = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, &dbCache, &vgInfo)); + + if (dbCache) { + vgHash = dbCache->vgInfo->vgHash; + } else { + vgHash = vgInfo->vgHash; + } + + if (tbMeta->tableType == TSDB_SUPER_TABLE) { + CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList)); + } else { + // USE HASH METHOD INSTEAD OF VGID IN TBMETA + ctgError("invalid method to get none stb vgInfo, tbType:%d", tbMeta->tableType); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + +#if 0 + int32_t vgId = tbMeta->vgId; + if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) { + ctgWarn("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); + CTG_ERR_JRET(TSDB_CODE_CTG_VG_META_MISMATCH); + } + + vgList = taosArrayInit(1, sizeof(SVgroupInfo)); + if (NULL == vgList) { + ctgError("taosArrayInit %d failed", (int32_t)sizeof(SVgroupInfo)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL == taosArrayPush(vgList, &vgroupInfo)) { + ctgError("taosArrayPush vgroupInfo to array failed, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *pVgList = vgList; + vgList = NULL; +#endif + } + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + taosMemoryFreeClear(tbMeta); + + if (vgInfo) { + taosHashCleanup(vgInfo->vgHash); + taosMemoryFreeClear(vgInfo); + } + + if (vgList) { + taosArrayDestroy(vgList); + vgList = NULL; + } + + CTG_RET(code); +} + int32_t catalogInit(SCatalogCfg *cfg) { if (gCtgMgmt.pCluster) { @@ -2090,7 +2271,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - atomic_store_8(&gCtgMgmt.exit, false); + atomic_store_8((int8_t*)&gCtgMgmt.exit, false); if (cfg) { memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); @@ -2125,14 +2306,15 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(ctgStartUpdateThread()); - tsem_init(&gCtgMgmt.sem, 0, 0); + tsem_init(&gCtgMgmt.queue.reqSem, 0, 0); + tsem_init(&gCtgMgmt.queue.rspSem, 0, 0); - gCtgMgmt.head = calloc(1, sizeof(SCtgQNode)); - if (NULL == gCtgMgmt.head) { + gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); + if (NULL == gCtgMgmt.queue.head) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - gCtgMgmt.tail = gCtgMgmt.head; + gCtgMgmt.queue.tail = gCtgMgmt.queue.head; qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); @@ -2161,7 +2343,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { return TSDB_CODE_SUCCESS; } - clusterCtg = calloc(1, sizeof(SCatalog)); + clusterCtg = taosMemoryCalloc(1, sizeof(SCatalog)); if (NULL == clusterCtg) { qError("calloc %d failed", (int32_t)sizeof(SCatalog)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -2269,7 +2451,7 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SArray** vgroupList) { +int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { @@ -2281,7 +2463,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c SArray *vgList = NULL; SHashObj *vgHash = NULL; SDBVgInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, forceUpdate, &dbCache, &vgInfo)); + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, &dbCache, &vgInfo)); if (dbCache) { vgHash = dbCache->vgInfo->vgHash; } else { @@ -2302,7 +2484,7 @@ _return: if (vgInfo) { taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } CTG_API_LEAVE(code); @@ -2315,37 +2497,37 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t code = 0; if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { + ctgFreeVgInfo(dbInfo); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG}; - SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - msg->dbId = dbId; - msg->dbInfo = dbInfo; + code = ctgPushUpdateVgMsgInQueue(pCtg, dbFName, dbId, dbInfo, false); - action.data = msg; +_return: - CTG_ERR_JRET(ctgPushAction(&action)); + CTG_API_LEAVE(code); +} - dbInfo = NULL; - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); +int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { + CTG_API_ENTER(); - CTG_API_LEAVE(code); + int32_t code = 0; -_return: + if (NULL == pCtg || NULL == dbFName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } - ctgFreeVgInfo(dbInfo); + if (NULL == pCtg->dbCache) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } - tfree(msg); + CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +_return: + CTG_API_LEAVE(code); } @@ -2353,12 +2535,12 @@ int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, } -int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { +int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { CTG_API_ENTER(); int32_t code = 0; - if (NULL == pCtg || NULL == dbFName) { + if (NULL == pCtg || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2366,15 +2548,34 @@ int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); + STableMeta *tblMeta = NULL; + int32_t exist = 0; + uint64_t dbId = 0; + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &exist, 0, &dbId)); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); + if (0 == exist) { + ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); + goto _return; + } + + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + if (TSDB_SUPER_TABLE == tblMeta->tableType) { + CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid, true)); + } else { + CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true)); + } + + _return: + taosMemoryFreeClear(tblMeta); + CTG_API_LEAVE(code); } + int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) { CTG_API_ENTER(); @@ -2388,7 +2589,7 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid)); + CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid, true)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); @@ -2420,7 +2621,7 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); @@ -2437,33 +2638,27 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; - SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - msg->output = output; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, false)); CTG_API_LEAVE(code); _return: - tfree(output->tbMeta); - tfree(output); - tfree(msg); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); CTG_API_LEAVE(code); } +int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName)); +} int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { CTG_API_ENTER(); @@ -2472,7 +2667,7 @@ int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL, true)); } int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { @@ -2492,83 +2687,28 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - - STableMeta *tbMeta = NULL; - int32_t code = 0; - SVgroupInfo vgroupInfo = {0}; - SCtgDBCache* dbCache = NULL; - SArray *vgList = NULL; - SDBVgInfo *vgInfo = NULL; - - *pVgList = NULL; - - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); - - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); - - SHashObj *vgHash = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, false, &dbCache, &vgInfo)); - - if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; - } else { - vgHash = vgInfo->vgHash; - } - - /* TODO REMOEV THIS .... - if (0 == tbMeta->vgId) { - SVgroupInfo vgroup = {0}; - - catalogGetTableHashVgroup(pCtg, pRpc, pMgmtEps, pTableName, &vgroup); - - tbMeta->vgId = vgroup.vgId; - } - // TODO REMOVE THIS ....*/ - if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList)); - } else { - int32_t vgId = tbMeta->vgId; - if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) { - ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } + int32_t code = 0; - vgList = taosArrayInit(1, sizeof(SVgroupInfo)); - if (NULL == vgList) { - ctgError("taosArrayInit %d failed", (int32_t)sizeof(SVgroupInfo)); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } + while (true) { + code = ctgGetTableDistVgInfo(pCtg, pRpc, pMgmtEps, pTableName, pVgList); + if (code) { + if (TSDB_CODE_CTG_VG_META_MISMATCH == code) { + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(CTG_FLAG_UNKNOWN_STB), NULL, true)); - if (NULL == taosArrayPush(vgList, &vgroupInfo)) { - ctgError("taosArrayPush vgroupInfo to array failed, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + CTG_ERR_JRET(ctgRefreshDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName)); + + continue; + } } - *pVgList = vgList; - vgList = NULL; + break; } _return: - if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); - } - - tfree(tbMeta); - - if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); - } - - if (vgList) { - taosArrayDestroy(vgList); - vgList = NULL; - } - CTG_API_LEAVE(code); } @@ -2587,7 +2727,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pM tNameGetFullDbName(pTableName, db); SDBVgInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, false, &dbCache, &vgInfo)); + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, &dbCache, &vgInfo)); CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); @@ -2600,7 +2740,7 @@ _return: if (vgInfo) { taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } CTG_API_LEAVE(code); @@ -2638,7 +2778,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); - tfree(pTableMeta); + taosMemoryFreeClear(pTableMeta); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -2656,7 +2796,7 @@ _return: int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); for (int32_t i = 0; i < aSize; ++i) { STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); - tfree(pMeta); + taosMemoryFreeClear(pMeta); } taosArrayDestroy(pRsp->pTableMeta); @@ -2705,13 +2845,14 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) void catalogDestroy(void) { qInfo("start to destroy catalog"); - if (NULL == gCtgMgmt.pCluster || atomic_load_8(&gCtgMgmt.exit)) { + if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) { return; } - atomic_store_8(&gCtgMgmt.exit, true); + atomic_store_8((int8_t*)&gCtgMgmt.exit, true); - tsem_post(&gCtgMgmt.sem); + tsem_post(&gCtgMgmt.queue.reqSem); + tsem_post(&gCtgMgmt.queue.rspSem); while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { taosUsleep(1); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index cc0e5bb1a9b653a72e766a32b75b6cc3710cf4c6..e62819b07857aaab9bdedfbdaee6cfb2826aa6f8 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -176,7 +176,7 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { output->ctbMeta.uid = 3; output->ctbMeta.suid = 2; - output->tbMeta = (STableMeta *)calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum)); + output->tbMeta = (STableMeta *)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum)); output->tbMeta->vgId = 9; output->tbMeta->tableType = TSDB_SUPER_TABLE; output->tbMeta->uid = 2; @@ -212,7 +212,7 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; - SDBVgInfo *dbVgroup = (SDBVgInfo *)calloc(1, sizeof(SDBVgInfo)); + SDBVgInfo *dbVgroup = (SDBVgInfo *)taosMemoryCalloc(1, sizeof(SDBVgInfo)); dbVgroup->vgVersion = vgVersion++; @@ -257,7 +257,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->tuid = ctgTestSuid + 1; rspMsg->vgId = 1; - rspMsg->pSchemas = (SSchema *)calloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); + rspMsg->pSchemas = (SSchema *)taosMemoryCalloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); SSchema *s = NULL; s = &rspMsg->pSchemas[0]; @@ -335,7 +335,7 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * metaRsp.suid = 0; metaRsp.tuid = ctgTestNormalTblUid++; metaRsp.vgId = 8; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -381,7 +381,7 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.suid = 0x0000000000000002; metaRsp.tuid = 0x0000000000000003; metaRsp.vgId = 9; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -428,7 +428,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.suid = ctgTestSuid; metaRsp.tuid = ctgTestSuid++; metaRsp.vgId = 0; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -477,7 +477,7 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp metaRsp.suid = ctgTestSuid + idx; metaRsp.tuid = ctgTestSuid + idx; metaRsp.vgId = 0; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -713,7 +713,7 @@ void *ctgTestGetDbVgroupThread(void *param) { int32_t n = 0; while (!ctgTestStop) { - code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, &vgList); if (code) { assert(0); } @@ -798,7 +798,7 @@ void *ctgTestGetCtableMetaThread(void *param) { assert(0); } - tfree(tbMeta); + taosMemoryFreeClear(tbMeta); if (ctgTestEnableSleep) { taosUsleep(taosRand() % 5); @@ -824,10 +824,10 @@ void *ctgTestSetCtableMetaThread(void *param) { action.act = CTG_ACT_UPDATE_TBL; while (!ctgTestStop) { - output = (STableMetaOutput *)malloc(sizeof(STableMetaOutput)); + output = (STableMetaOutput *)taosMemoryMalloc(sizeof(STableMetaOutput)); ctgTestBuildCTableMetaOutput(output); - SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)malloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); msg->pCtg = pCtg; msg->output = output; action.data = msg; @@ -933,7 +933,7 @@ TEST(tableMeta, normalTable) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -941,7 +941,7 @@ TEST(tableMeta, normalTable) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1015,7 +1015,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); strcpy(n.tname, ctgTestSTablename); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); @@ -1042,7 +1042,7 @@ TEST(tableMeta, childTableCase) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -1050,7 +1050,7 @@ TEST(tableMeta, childTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1164,7 +1164,7 @@ TEST(tableMeta, superTableCase) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -1173,7 +1173,7 @@ TEST(tableMeta, superTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1307,14 +1307,14 @@ TEST(tableMeta, updateStbMeta) { } - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); STableMetaRsp rsp = {0}; ctgTestBuildSTableMetaRsp(&rsp); code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); - tfree(rsp.pSchemas); + taosMemoryFreeClear(rsp.pSchemas); while (true) { uint64_t n = 0; @@ -1345,7 +1345,7 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1 + 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); @@ -1407,7 +1407,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1424,7 +1424,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1486,7 +1486,7 @@ TEST(refreshGetMeta, normal2notexist) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1560,7 +1560,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1576,7 +1576,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1645,7 +1645,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1662,7 +1662,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1730,7 +1730,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1748,7 +1748,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1816,7 +1816,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1835,7 +1835,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -2009,7 +2009,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + code = catalogGetDBVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); @@ -2094,14 +2094,14 @@ TEST(multiThread, getSetRmSameDbVgroup) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); taosSsleep(1); - pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2146,14 +2146,14 @@ TEST(multiThread, getSetRmDiffDbVgroup) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); taosSsleep(1); - pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2198,13 +2198,13 @@ TEST(multiThread, ctableMeta) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); taosSsleep(1); - pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); + taosThreadCreate(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2275,7 +2275,7 @@ TEST(rentTest, allRent) { printf("%d - expired dbNum:%d\n", i, num); if (dbs) { printf("%d - expired dbId:%" PRId64 ", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } @@ -2287,7 +2287,7 @@ TEST(rentTest, allRent) { printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } - free(stable); + taosMemoryFree(stable); stable = NULL; } printf("*************************************************\n"); diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 8acb6f7e8d933153c4d35437529dfaf46acf6174..85356a862ce282ac53aaad4ee72f0a77b19f115c 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -29,7 +29,7 @@ struct SDataSinkHandle; typedef struct SDataSinkManager { SDataSinkMgtCfg cfg; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 833ac13226288a40d783ecfc24f279cf694a8c5b..ead830394eaeda9acd64847f3fb2695f0ed0fffa 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -76,11 +76,12 @@ typedef struct SResultRowCell { * If the number of generated results is greater than this value, * query query will be halt and return results to client immediate. */ -typedef struct SRspResultInfo { - int64_t total; // total generated result size in rows - int32_t capacity; // capacity of current result output buffer - int32_t threshold; // result size threshold in rows. -} SRspResultInfo; +typedef struct SResultInfo { // TODO refactor + int64_t totalRows; // total generated result size in rows + int64_t totalBytes; // total results in bytes. + int32_t capacity; // capacity of current result output buffer + int32_t threshold; // result size threshold in rows. +} SResultInfo; typedef struct SColumnFilterElem { int16_t bytes; // column length @@ -160,8 +161,8 @@ typedef struct STaskCostInfo { typedef struct SOperatorCostInfo { uint64_t openCost; uint64_t execCost; - uint64_t totalRows; - uint64_t totalBytes; +// uint64_t totalRows; +// uint64_t totalBytes; } SOperatorCostInfo; typedef struct { @@ -242,6 +243,8 @@ typedef struct STaskAttr { struct SOperatorInfo; +typedef void (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, char **result, int32_t *length); +typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char *result, int32_t length); typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* param); typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* param, bool* newgroup); typedef void (*__optr_close_fn_t)(void* param, int32_t num); @@ -253,11 +256,6 @@ typedef struct STaskIdInfo { char* str; } STaskIdInfo; -typedef struct STaskBufInfo { - int32_t bufSize; // total available buffer size in bytes - int32_t remainBuf; // remain buffer size -} STaskBufInfo; - typedef struct SExecTaskInfo { STaskIdInfo id; char* content; @@ -269,8 +267,7 @@ typedef struct SExecTaskInfo { uint64_t totalRows; // total number of rows STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure char* sql; // query sql string - jmp_buf env; // when error occurs, abort - STaskBufInfo bufInfo; // available buffer info this task + jmp_buf env; // struct SOperatorInfo* pRoot; } SExecTaskInfo; @@ -307,7 +304,7 @@ typedef struct STaskRuntimeEnv { int64_t currentOffset; // dynamic offset value STableQueryInfo* current; - SRspResultInfo resultInfo; + SResultInfo resultInfo; SHashObj* pTableRetrieveTsMap; struct SUdfInfo* pUdfInfo; } STaskRuntimeEnv; @@ -315,7 +312,6 @@ typedef struct STaskRuntimeEnv { enum { OP_NOT_OPENED = 0x0, OP_OPENED = 0x1, - OP_IN_EXECUTING = 0x3, OP_RES_TO_RETURN = 0x5, OP_EXEC_DONE = 0x9, }; @@ -331,13 +327,15 @@ typedef struct SOperatorInfo { STaskRuntimeEnv* pRuntimeEnv; // todo remove it SExecTaskInfo* pTaskInfo; SOperatorCostInfo cost; - + SResultInfo resultInfo; struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator __optr_fn_t getNextFn; __optr_fn_t cleanupFn; __optr_close_fn_t closeFn; __optr_open_fn_t _openFn; // DO NOT invoke this function directly + __optr_encode_fn_t encodeResultRow; // + __optr_decode_fn_t decodeResultRow; } SOperatorInfo; typedef struct { @@ -356,7 +354,7 @@ typedef struct SQInfo { STaskAttr query; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; - pthread_mutex_t lock; // used to synchronize the rsp/query threads + TdThreadMutex lock; // used to synchronize the rsp/query threads tsem_t ready; int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context @@ -366,9 +364,9 @@ typedef struct SQInfo { } SQInfo; enum { - EX_SOURCE_DATA_NOT_READY = 0x1, - EX_SOURCE_DATA_READY = 0x2, - EX_SOURCE_DATA_EXHAUSTED = 0x3, + DATA_NOT_READY = 0x1, + DATA_READY = 0x2, + DATA_EXHAUSTED = 0x3, }; typedef struct SSourceDataInfo { @@ -379,6 +377,18 @@ typedef struct SSourceDataInfo { int32_t status; } SSourceDataInfo; +typedef struct SLoadRemoteDataInfo { + uint64_t totalSize; // total load bytes from remote + uint64_t totalRows; // total number of rows + uint64_t totalElapsed; // total elapsed time +} SLoadRemoteDataInfo; + +enum { + EX_SOURCE_DATA_NOT_READY = 0x1, + EX_SOURCE_DATA_READY = 0x2, + EX_SOURCE_DATA_EXHAUSTED = 0x3, +}; + typedef struct SExchangeInfo { SArray* pSources; SArray* pSourceDataInfo; @@ -387,9 +397,7 @@ typedef struct SExchangeInfo { SSDataBlock* pResult; bool seqLoadData; // sequential load data or not, false by default int32_t current; - uint64_t totalSize; // total load bytes from remote - uint64_t totalRows; // total number of rows - uint64_t totalElapsed; // total elapsed time + SLoadRemoteDataInfo loadInfo; } SExchangeInfo; typedef struct STableScanInfo { @@ -422,6 +430,8 @@ typedef struct STagScanInfo { typedef struct SStreamBlockScanInfo { SSDataBlock* pRes; // result SSDataBlock + int32_t blockType; // current block type + bool blockValid; // Is current data has returned? SColumnInfo* pCols; // the output column info uint64_t numOfRows; // total scanned rows uint64_t numOfExec; // execution times @@ -434,19 +444,23 @@ typedef struct SSysTableScanInfo { void* readHandle; }; - void *pCur; // cursor - SRetrieveTableReq* pReq; - SEpSet epSet; - int32_t type; // show type - tsem_t ready; - SSchema* pSchema; - SSDataBlock* pRes; - - int32_t capacity; - int64_t numOfBlocks; // extract basic running information. - int64_t totalRows; - int64_t elapsedTime; - int64_t totalBytes; + SRetrieveMetaTableRsp *pRsp; + SRetrieveTableReq req; + SEpSet epSet; + tsem_t ready; + + int32_t accountId; + bool showRewrite; + SNode* pCondition; // db_name filter condition, to discard data that are not in current database + void *pCur; // cursor for iterate the local table meta store. + SArray *scanCols; // SArray scan column id list + + int32_t type; // show type, TODO remove it + SName name; + SSDataBlock* pRes; + int32_t capacity; + int64_t numOfBlocks; // extract basic running information. + SLoadRemoteDataInfo loadInfo; } SSysTableScanInfo; typedef struct SOptrBasicInfo { @@ -491,10 +505,9 @@ typedef struct SAggOperatorInfo { } SAggOperatorInfo; typedef struct SProjectOperatorInfo { - SOptrBasicInfo binfo; - SSDataBlock *existDataBlock; - int32_t threshold; - bool hasVarCol; + SOptrBasicInfo binfo; + SSDataBlock *existDataBlock; + int32_t threshold; } SProjectOperatorInfo; typedef struct SLimitOperatorInfo { @@ -533,12 +546,26 @@ typedef struct SFillOperatorInfo { void** p; SSDataBlock* existNewGroupBlock; bool multigroupResult; + SInterval intervalInfo; + int32_t capacity; } SFillOperatorInfo; +typedef struct SGroupKeys { + char *pData; + bool isNull; + int16_t type; + int32_t bytes; +}SGroupKeys; + typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; - int32_t colIndex; - char* prevData; // previous group by value + SArray* pGroupCols; + SArray* pGroupColVals; // current group column values, SArray + bool isInit; // denote if current val is initialized or not + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SGroupResInfo groupResInfo; + SAggSupporter aggSup; } SGroupbyOperatorInfo; typedef struct SSessionAggOperatorInfo { @@ -629,31 +656,31 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprI SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, - int32_t tableType, SEpSet epset, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfDownstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, + SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); +SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, + SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); +SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, + int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput, bool multigroupResult); -SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); - SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); + SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows, void* merger); SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, @@ -667,7 +694,6 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput); -// int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); @@ -677,6 +703,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); + void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity); @@ -690,16 +717,9 @@ void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win); STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); -int32_t buildArithmeticExprFromMsg(SExprInfo* pArithExprInfo, void* pQueryMsg); - bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); -bool checkNeedToCompressQueryCol(SQInfo* pQInfo); -void setQueryStatus(STaskRuntimeEnv* pRuntimeEnv, int8_t status); -int32_t doDumpQueryResult(SQInfo* pQInfo, char* data, int8_t compressed, int32_t* compLen); - -size_t getResultSize(SQInfo* pQInfo, int64_t* numOfRows); void setTaskKilled(SExecTaskInfo* pTaskInfo); void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType); @@ -709,8 +729,6 @@ void calculateOperatorProfResults(SQInfo* pQInfo); void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); -void freeQueryAttr(STaskAttr* pQuery); - int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx* pCtx, int32_t idx, int32_t type); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a0ee048d82051bb35e3f960f08aceb24398f1488..34894c235b0e0565710a2b8d1908ff5a6b6e3fee 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -15,11 +15,12 @@ #include "dataSinkInt.h" #include "dataSinkMgt.h" +#include "executorimpl.h" #include "planner.h" #include "tcompression.h" #include "tglobal.h" #include "tqueue.h" -#include "executorimpl.h" +#include "tdatablock.h" typedef struct SDataDispatchBuf { int32_t useSize; @@ -43,7 +44,7 @@ typedef struct SDataDispatchHandle { int32_t status; bool queryEnd; uint64_t useconds; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SDataDispatchHandle; static bool needCompress(const SSDataBlock* pData, const SDataBlockDescNode* pSchema) { @@ -64,47 +65,68 @@ static bool needCompress(const SSDataBlock* pData, const SDataBlockDescNode* pSc } static int32_t compressColData(SColumnInfoData *pColRes, int32_t numOfRows, char *data, int8_t compressed) { - int32_t colSize = pColRes->info.bytes * numOfRows; + int32_t colSize = colDataGetLength(pColRes, numOfRows); return (*(tDataTypes[pColRes->info.type].compFunc))( pColRes->pData, colSize, numOfRows, data, colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); } -static void copyData(const SInputData* pInput, const SDataBlockDescNode* pSchema, char* data, int8_t compressed, int32_t *compLen) { +static void copyData(const SInputData* pInput, const SDataBlockDescNode* pSchema, char* data, int8_t compressed, int32_t * dataLen) { int32_t numOfCols = LIST_LENGTH(pSchema->pSlots); - int32_t *compSizes = (int32_t*)data; - if (compressed) { - data += numOfCols * sizeof(int32_t); - } + int32_t * colSizes = (int32_t*)data; + + data += numOfCols * sizeof(int32_t); + *dataLen = (numOfCols * sizeof(int32_t)); + int32_t numOfRows = pInput->pData->info.rows; for (int32_t col = 0; col < numOfCols; ++col) { SColumnInfoData* pColRes = taosArrayGet(pInput->pData->pDataBlock, col); + + // copy the null bitmap + if (IS_VAR_DATA_TYPE(pColRes->info.type)) { + size_t metaSize = numOfRows * sizeof(int32_t); + memcpy(data, pColRes->varmeta.offset, metaSize); + data += metaSize; + (*dataLen) += metaSize; + } else { + int32_t len = BitmapLen(numOfRows); + memcpy(data, pColRes->nullbitmap, len); + data += len; + (*dataLen) += len; + } + if (compressed) { - compSizes[col] = compressColData(pColRes, pInput->pData->info.rows, data, compressed); - data += compSizes[col]; - *compLen += compSizes[col]; - compSizes[col] = htonl(compSizes[col]); + colSizes[col] = compressColData(pColRes, numOfRows, data, compressed); + data += colSizes[col]; + (*dataLen) += colSizes[col]; } else { - memmove(data, pColRes->pData, pColRes->info.bytes * pInput->pData->info.rows); - data += pColRes->info.bytes * pInput->pData->info.rows; + colSizes[col] = colDataGetLength(pColRes, numOfRows); + (*dataLen) += colSizes[col]; + memmove(data, pColRes->pData, colSizes[col]); + data += colSizes[col]; } + + colSizes[col] = htonl(colSizes[col]); } } -// data format with compress: SDataCacheEntry | cols_data_offset | col1_data col2_data ... | numOfTables | STableIdInfo STableIdInfo ... -// data format: SDataCacheEntry | col1_data col2_data ... | numOfTables | STableIdInfo STableIdInfo ... +// data format: +// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... +// | | sizeof(int32_t) * numOfCols | actual size | | actual size | | +// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+ +// The length of bitmap is decided by number of rows of this data block, and the length of each column data is +// recorded in the first segment, next to the struct header static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = (int8_t)needCompress(pInput->pData, pHandle->pSchema); - pEntry->numOfRows = pInput->pData->info.rows; - pEntry->dataLen = 0; + pEntry->numOfRows = pInput->pData->info.rows; + pEntry->dataLen = 0; pBuf->useSize = sizeof(SRetrieveTableRsp); copyData(pInput, pHandle->pSchema, pEntry->data, pEntry->compressed, &pEntry->dataLen); - if (0 == pEntry->compressed) { - pEntry->dataLen = pHandle->pSchema->resultRowSize * pInput->pData->info.rows; - } - pBuf->useSize += pEntry->dataLen; - // todo completed + + pEntry->dataLen = pEntry->dataLen; + pBuf->useSize += pEntry->dataLen; } static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { @@ -115,9 +137,12 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, return false; } - // struct size + data payload + length for each column - pBuf->allocSize = sizeof(SRetrieveTableRsp) + pDispatcher->pSchema->resultRowSize * pInput->pData->info.rows + pInput->pData->info.numOfCols * sizeof(int32_t); - pBuf->pData = malloc(pBuf->allocSize); + // NOTE: there are four bytes of an integer more than the required buffer space. + // struct size + data payload + length for each column + bitmap length + pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockDataGetSerialMetaSize(pInput->pData) + + ceil(blockDataGetSerialRowSize(pInput->pData) * pInput->pData->info.rows); + + pBuf->pData = taosMemoryMalloc(pBuf->allocSize); if (pBuf->pData == NULL) { qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); } @@ -126,19 +151,19 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, } static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); int32_t blockNums = taosQueueSize(pDispatcher->pDataBlocks); int32_t status = (0 == blockNums ? DS_BUF_EMPTY : (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); pDispatcher->status = status; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return status; } static int32_t getStatus(SDataDispatchHandle* pDispatcher) { - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); int32_t status = pDispatcher->status; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return status; } @@ -156,10 +181,10 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); pDispatcher->queryEnd = true; pDispatcher->useconds = useconds; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); } static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { @@ -169,6 +194,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE *pLen = 0; return; } + SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); @@ -189,31 +215,31 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; pOutput->compressed = pEntry->compressed; - tfree(pDispatcher->nextOutput.pData); // todo persistent + taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->pSchema->precision; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return TSDB_CODE_SUCCESS; } static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - tfree(pDispatcher->nextOutput.pData); + taosMemoryFreeClear(pDispatcher->nextOutput.pData); while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); - tfree(pBuf->pData); + taosMemoryFreeClear(pBuf->pData); taosFreeQitem(pBuf); } taosCloseQueue(pDispatcher->pDataBlocks); - pthread_mutex_destroy(&pDispatcher->mutex); + taosThreadMutexDestroy(&pDispatcher->mutex); } int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { - SDataDispatchHandle* dispatcher = calloc(1, sizeof(SDataDispatchHandle)); + SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -228,7 +254,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->status = DS_BUF_EMPTY; dispatcher->queryEnd = false; dispatcher->pDataBlocks = taosOpenQueue(); - pthread_mutex_init(&dispatcher->mutex, NULL); + taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 4e8583eb2ab6e4871eec727500cf8ff800a80b41..997397314fc253d55530d9729c3961cc1357950b 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -22,7 +22,7 @@ static SDataSinkManager gDataSinkManager = {0}; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg) { gDataSinkManager.cfg = *cfg; - pthread_mutex_init(&gDataSinkManager.mutex, NULL); + taosThreadMutexInit(&gDataSinkManager.mutex, NULL); return 0; // to avoid compiler eror } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a04a10ef95242ef59b74001a1ea07d7cc004fd24..f0cffafca2fcc053969707a181832a5d005b7124 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -58,8 +58,8 @@ int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->curPos = -1; pResultRowInfo->capacity = size; - pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); - pResultRowInfo->pPosition = calloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); + pResultRowInfo->pResult = taosMemoryCalloc(pResultRowInfo->capacity, POINTER_BYTES); + pResultRowInfo->pPosition = taosMemoryCalloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); if (pResultRowInfo->pResult == NULL || pResultRowInfo->pPosition == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -79,11 +79,11 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { for(int32_t i = 0; i < pResultRowInfo->size; ++i) { if (pResultRowInfo->pResult[i]) { - tfree(pResultRowInfo->pResult[i]->key); + taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); } } - tfree(pResultRowInfo->pResult); + taosMemoryFreeClear(pResultRowInfo->pResult); } void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { @@ -163,7 +163,7 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { pResultRow->offset = -1; pResultRow->closed = false; - tfree(pResultRow->key); + taosMemoryFreeClear(pResultRow->key); pResultRow->win = TSWINDOW_INITIALIZER; } @@ -190,7 +190,7 @@ SResultRow* getNewResultRow(SResultRowPool* p) { void* ptr = NULL; if (p->position.pos == 0) { - ptr = calloc(1, p->blockSize); + ptr = taosMemoryCalloc(1, p->blockSize); taosArrayPush(p->pData, &ptr); } else { @@ -403,8 +403,8 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); } - posList = calloc(size, sizeof(int32_t)); - pTableQueryInfoList = malloc(POINTER_BYTES * size); + posList = taosMemoryCalloc(size, sizeof(int32_t)); + pTableQueryInfoList = taosMemoryMalloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { // qError("QInfo:%"PRIu64" failed alloc memory", GET_TASKID(pRuntimeEnv)); @@ -483,9 +483,9 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv // pGroupResInfo->currentGroup, endt - startt); _end: - tfree(pTableQueryInfoList); - tfree(posList); - tfree(pTree); + taosMemoryFreeClear(pTableQueryInfoList); + taosMemoryFreeClear(posList); + taosMemoryFreeClear(pTree); return code; } @@ -529,7 +529,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // // compress extra bytes // size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; -// char* tmp = malloc(x + 2); +// char* tmp = taosMemoryMalloc(x + 2); // // bool comp = false; // int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); @@ -547,7 +547,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // } else { // tbufWriteBinary(bw, p, len); // } -// tfree(tmp); +// taosMemoryFreeClear(tmp); //} //void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { @@ -570,7 +570,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // char* outputBuf = NULL; // if (comp) { -// outputBuf = malloc(originalLen); +// outputBuf = taosMemoryMalloc(originalLen); // // size_t actualLen = compLen; // const char* compStr = tbufReadBinary(&br, &actualLen); @@ -584,7 +584,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); // if (comp) { -// tfree(outputBuf); +// taosMemoryFreeClear(outputBuf); // } //} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a8602b7c774469a3d2117877c6d0baeacb12345a..e89bc5df0e52940e19bfe69c3d17b46bd79edb02 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -16,9 +16,9 @@ #include "executor.h" #include "executorimpl.h" #include "planner.h" -#include "tq.h" +#include "vnode.h" -static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) { +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t type, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { @@ -30,19 +30,40 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) qError("join not supported for stream block scan, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } - - return doSetStreamBlock(pOperator->pDownstream[0], input, id); + pOperator->status = OP_NOT_OPENED; + return doSetStreamBlock(pOperator->pDownstream[0], input, type, id); } else { SStreamBlockScanInfo* pInfo = pOperator->info; - if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) { - qError("submit msg messed up when initing stream block, %s" PRIx64, id); + + // the block type can not be changed in the streamscan operators + if (pInfo->blockType == 0) { + pInfo->blockType = type; + } else if (pInfo->blockType != type) { return TSDB_CODE_QRY_APP_ERROR; } + + if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) { + qError("submit msg messed up when initing stream block, %s" PRIx64, id); + return TSDB_CODE_QRY_APP_ERROR; + } + } else { + ASSERT(!pInfo->blockValid); + + SSDataBlock* pDataBlock = input; + pInfo->pRes->info = pDataBlock->info; + taosArrayClear(pInfo->pRes->pDataBlock); + taosArrayAddAll(pInfo->pRes->pDataBlock, pDataBlock->pDataBlock); + + // set current block valid. + pInfo->blockValid = true; + } + return TSDB_CODE_SUCCESS; } } -int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input) { +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type) { if (tinfo == NULL) { return TSDB_CODE_QRY_APP_ERROR; } @@ -53,7 +74,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void*)input, GET_TASKID(pTaskInfo)); + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void*)input, type, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { @@ -99,7 +120,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA // traverse to the streamscan node to add this table id SOperatorInfo* pInfo = pTaskInfo->pRoot; - while(pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + while (pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { pInfo = pInfo->pDownstream[0]; } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 7e55a4b3e1615f8e98e78d7e27449a1a99e4de3b..cc9921ce73c655508a395733a43170d78c0ebf3a 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -30,7 +30,7 @@ #include "query.h" typedef struct STaskMgmt { - pthread_mutex_t lock; + TdThreadMutex lock; SCacheObj *qinfoPool; // query handle pool int32_t vgId; bool closed; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 947fb08ff907421787972c78f8e313f54f48d1a7..4f7f3d2d7ea9e056b688b2e1e324830c3be4b395 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,8 +13,11 @@ * along with this program. If not, see . */ +#include #include #include +#include +#include #include #include "os.h" @@ -70,7 +73,7 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { if (v % 1000 <= 0) { return NULL; } else { - return malloc(__size); + return taosMemoryMalloc(__size); } } @@ -79,7 +82,7 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { if (v % 1000 <= 0) { return NULL; } else { - return calloc(num, __size); + return taosMemoryCalloc(num, __size); } } @@ -88,7 +91,7 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { if (v % 5 <= 1) { return NULL; } else { - return realloc(p, __size); + return taosMemoryRealloc(p, __size); } } @@ -212,6 +215,7 @@ static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); + static void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput); static void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput); static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput); @@ -239,12 +243,12 @@ static void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset); static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); -static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); - +static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupId, + SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); -static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); +static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); @@ -316,7 +320,7 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, STaskRuntimeEn SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { const static int32_t minSize = 8; - SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = numOfOutput; res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfOutput; ++i) { @@ -326,38 +330,17 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO idata.info.colId = pExpr[i].base.resSchema.colId; int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); - idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform - taosArrayPush(res->pDataBlock, &idata); - } - - return res; -} - -SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) { - size_t numOfOutput = taosArrayGetSize(pExprInfo); - - SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = numOfOutput; - res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - - for (int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData idata = {{0}}; - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); - - idata.info.type = pExpr->base.resSchema.type; - idata.info.bytes = pExpr->base.resSchema.bytes; - idata.info.colId = pExpr->base.resSchema.colId; + idata.pData = taosMemoryCalloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } - blockDataEnsureCapacity(res, numOfRows); return res; } SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->info.numOfCols = numOfCols; pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); @@ -440,12 +423,12 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) newCapacity += 4; } - char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + char *t = taosMemoryRealloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pResultRowInfo->pPosition = realloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); + pResultRowInfo->pPosition = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); pResultRowInfo->pResult = (SResultRow **)t; int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; @@ -691,12 +674,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T int64_t key = w->skey; while(key < ts) { // moving towards end - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); - } else { - key += pInterval->sliding; - } - + key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); if (key >= ts) { break; } @@ -712,12 +690,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true); - - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - w.ekey = w.skey + pInterval->interval - 1; - } + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; } @@ -739,7 +712,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t } w.skey = st; - w.ekey = w.skey + pInterval->interval - 1; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } } return w; @@ -1030,8 +1003,8 @@ static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t of pCtx[k].startTs = pWin->skey; // keep it temporarialy - int32_t startOffset = pCtx[k].input.startRowIndex; bool hasAgg = pCtx[k].input.colDataAggIsSet; + int32_t startOffset = pCtx[k].input.startRowIndex; int32_t numOfRows = pCtx[k].input.numOfRows; int32_t pos = (order == TSDB_ORDER_ASC) ? offset : offset - (forwardStep - 1); @@ -1234,13 +1207,6 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, ASSERT(pCtx[i].input.pData[0] != NULL); - // if (pCtx[i].functionId < 0) { - // SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); - // pCtx[i].ptsList = (int64_t*) tsInfo->pData; - - // continue; - // } - // uint32_t status = aAggs[pCtx[i].functionId].status; // if ((status & (FUNCSTATE_SELECTIVITY | FUNCSTATE_NEED_TS)) != 0) { // SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); @@ -1275,16 +1241,17 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunction } } -static void projectApplyFunctions(SqlFunctionCtx *pCtx, int32_t numOfOutput) { +static void projectApplyFunctions(SSDataBlock* pResult, SqlFunctionCtx *pCtx, int32_t numOfOutput) { for (int32_t k = 0; k < numOfOutput; ++k) { - // Always set the asc order for merge stage process - if (pCtx[k].currentStage == MERGE_STAGE) { - pCtx[k].order = TSDB_ORDER_ASC; + if (pCtx[k].fpSet.init == NULL) { // it is a project query + SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, k); + colDataAssign(pColInfoData, pCtx[k].input.pData[0], pCtx[k].input.numOfRows); + } else { // TODO: arithmetic and other process. + ASSERT(0); } -// pCtx[k].fpSet.process(&pCtx[k]); -// aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); -// } } + + pResult->info.rows = pCtx[0].input.numOfRows; } void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, @@ -1646,88 +1613,176 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); } +static bool groupKeyCompare(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { + SColumnDataAgg* pColAgg = NULL; + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + if (pBlock->pBlockAgg != NULL) { + pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + bool isNull = colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg); -static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - STableQueryInfo* item = pRuntimeEnv->current; + SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); + if (pkey->isNull && isNull) { + continue; + } - SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); + if (isNull || pkey->isNull) { + return false; + } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; + char* val = colDataGetData(pColInfoData, rowIndex); - if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); - return; + if (IS_VAR_DATA_TYPE(pkey->type)) { + int32_t len = varDataLen(val); + if (len == varDataLen(pkey->pData) && memcmp(varDataVal(pkey->pData), varDataVal(val), len) == 0) { + continue; + } else { + return false; + } + } else { + if (memcmp(pkey->pData, val, pkey->bytes) != 0) { + return false; + } + } } - SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); - int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; + return true; +} - STimeWindow w = TSWINDOW_INITIALIZER; +static void keepGroupKeys(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { + SColumnDataAgg* pColAgg = NULL; - int32_t num = 0; - for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { - char* val = ((char*)pColInfoData->pData) + bytes * j; - if (isNull(val, type)) { - continue; - } + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); - // Compare with the previous row of this column, and do not set the output buffer again if they are identical. - if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); - memcpy(pInfo->prevData, val, bytes); - num++; - continue; + if (pBlock->pBlockAgg != NULL) { + pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } - if (IS_VAR_DATA_TYPE(type)) { - int32_t len = varDataLen(val); - if(len == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), len) == 0) { - num++; - continue; - } + SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + pkey->isNull = true; } else { - if (memcmp(pInfo->prevData, val, bytes) == 0) { - num++; - continue; + char* val = colDataGetData(pColInfoData, rowIndex); + if (IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, varDataTLen(val)); + } else { + memcpy(pkey->pData, val, pkey->bytes); } } + } +} - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); +static int32_t generatedHashKey(void* pKey, int32_t* length, SArray* pGroupColVals) { + ASSERT(pKey != NULL); + size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); + + char* isNull = (char*) pKey; + char* pStart = (char*) pKey + sizeof(int8_t) * numOfGroupCols; + for(int32_t i = 0; i < numOfGroupCols; ++i) { + SGroupKeys* pkey = taosArrayGet(pGroupColVals, i); + if (pkey->isNull) { + isNull[i] = 1; + continue; } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + isNull[i] = 0; + if (IS_VAR_DATA_TYPE(pkey->type)) { + varDataCopy(pStart, pkey->pData); + pStart += varDataTLen(pkey->pData); + ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); + } else { + memcpy(pStart, pkey->pData, pkey->bytes); + pStart += pkey->bytes; } + } -// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + *length = (pStart - (char*) pKey); + return 0; +} - num = 1; - memcpy(pInfo->prevData, val, bytes); +// assign the group keys or user input constant values if required +static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) { + for(int32_t i = 0; i < numOfOutput; ++i) { + if (pCtx[i].functionId == -1) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]); + + SColumnInfoData* pColInfoData = pCtx[i].input.pData[0]; + if (!colDataIsNull(pColInfoData, totalRows, rowIndex, NULL)) { + char* dest = GET_ROWCELL_INTERBUF(pEntryInfo); + char* data = colDataGetData(pColInfoData, rowIndex); + + // set result exists, todo refactor + memcpy(dest, data, pColInfoData->info.bytes); + pEntryInfo->hasResult = DATA_SET_FLAG; + pEntryInfo->numOfRes = 1; + } + } } +} - if (num > 0) { - char* val = ((char*)pColInfoData->pData) + bytes * (pSDataBlock->info.rows - num); - memcpy(pInfo->prevData, val, bytes); +static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock *pBlock) { + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + SGroupbyOperatorInfo *pInfo = pOperator->info; - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); + SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; + int32_t numOfGroupCols = taosArrayGetSize(pInfo->pGroupCols); +// if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); +// return; +// } + + int32_t len = 0; + STimeWindow w = TSWINDOW_INITIALIZER; + + int32_t num = 0; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + // Compare with the previous row of this column, and do not set the output buffer again if they are identical. + if (!pInfo->isInit) { + keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); + pInfo->isInit = true; + num++; + continue; + } + + bool equal = groupKeyCompare(pInfo, pBlock, j, numOfGroupCols); + if (equal) { + num++; + continue; } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); + /*int32_t ret = */generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); + int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, + pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } -// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + int32_t rowIndex = j - num; + doApplyFunctions(pCtx, &w, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + + // assign the group keys or user input constant values if required + doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); + keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); + num = 1; } - tfree(pInfo->prevData); + if (num > 0) { + /*int32_t ret = */generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); + int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, + pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + int32_t rowIndex = pBlock->info.rows - num; + doApplyFunctions(pCtx, &w, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); + } } static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { @@ -1802,7 +1857,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { if (IS_VAR_DATA_TYPE(type)) { if (pResultRow->key == NULL) { - pResultRow->key = malloc(varDataTLen(pData)); + pResultRow->key = taosMemoryMalloc(varDataTLen(pData)); varDataCopy(pResultRow->key, pData); } else { assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); @@ -1816,35 +1871,18 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { } } -static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - SDiskbasedBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; +static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupId, + SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup) { SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx *pCtx = binfo->pCtx; - // not assign result buffer yet, add new result buffer, TODO remove it - char* d = pData; - int16_t len = bytes; - if (IS_VAR_DATA_TYPE(type)) { - d = varDataVal(pData); - len = varDataLen(pData); - } - - int64_t tid = 0; - SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, d, len, true, groupIndex); + SResultRow *pResultRow = doSetResultOutBufByKey_rv(pBuf, pResultRowInfo, groupId, (char *)pData, bytes, true, groupId, + pTaskInfo, true, pAggSup); assert (pResultRow != NULL); setResultRowKey(pResultRow, pData, type); - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQueryAttr->resultRowSize); - if (ret != 0) { - return -1; - } - } - setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset); - initCtxOutputBuffer(pCtx, numOfCols); + setResultRowOutputBufInitCtx_rv(pBuf, pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); return TSDB_CODE_SUCCESS; } @@ -1875,6 +1913,10 @@ static bool functionNeedToExecute(SqlFunctionCtx *pCtx) { // in case of timestamp column, always generated results. int32_t functionId = pCtx->functionId; + if (functionId == -1) { + return false; + } + if (functionId == FUNCTION_TS) { return true; } @@ -1934,7 +1976,7 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { int16_t tagLen = 0; SqlFunctionCtx* p = NULL; - SqlFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + SqlFunctionCtx** pTagCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); if (pTagCtx == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -1960,7 +2002,7 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { p->subsidiaryRes.numOfCols = num; p->subsidiaryRes.bufLen = tagLen; } else { - tfree(pTagCtx); + taosMemoryFreeClear(pTagCtx); } return TSDB_CODE_SUCCESS; @@ -1970,14 +2012,14 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI int32_t** rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } - *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + *rowCellInfoOffset = taosMemoryCalloc(numOfOutput, sizeof(int32_t)); if (*rowCellInfoOffset == 0) { - tfree(pFuncCtx); + taosMemoryFreeClear(pFuncCtx); return NULL; } @@ -2068,14 +2110,14 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI } static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset) { - SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } - *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + *rowCellInfoOffset = taosMemoryCalloc(numOfOutput, sizeof(int32_t)); if (*rowCellInfoOffset == 0) { - tfree(pFuncCtx); + taosMemoryFreeClear(pFuncCtx); return NULL; } @@ -2085,11 +2127,20 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num SExprBasicInfo *pFunct = &pExpr->base; SqlFunctionCtx* pCtx = &pFuncCtx[i]; - fmGetFuncExecFuncs(pExpr->pExpr->_function.pFunctNode->funcId, &pCtx->fpSet); - pCtx->input.numOfInputCols = pFunct->numOfParams; + if (pExpr->pExpr->_function.pFunctNode != NULL) { + SFuncExecEnv env = {0}; + pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; + + fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); + pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); + pCtx->resDataInfo.interBufSize = env.calcMemSize; + } else { + pCtx->functionId = -1; + } - pCtx->input.pData = calloc(pFunct->numOfParams, POINTER_BYTES); - pCtx->input.pColumnDataAgg = calloc(pFunct->numOfParams, POINTER_BYTES); + pCtx->input.numOfInputCols = pFunct->numOfParams; + pCtx->input.pData = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); + pCtx->input.pColumnDataAgg = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); pCtx->ptsOutputBuf = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; @@ -2097,10 +2148,6 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num pCtx->order = TSDB_ORDER_ASC; pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; - - SFuncExecEnv env = {0}; - pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); - pCtx->resDataInfo.interBufSize = env.calcMemSize; #if 0 for (int32_t j = 0; j < pCtx->numOfParams; ++j) { // int16_t type = pFunct->param[j].nType; @@ -2151,7 +2198,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num } for(int32_t i = 1; i < numOfOutput; ++i) { - (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i].resDataInfo.interBufSize); + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i - 1].resDataInfo.interBufSize); } setCtxTagColumnInfo(pFuncCtx, numOfOutput); @@ -2169,10 +2216,10 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } taosVariantDestroy(&pCtx[i].tag); - tfree(pCtx[i].subsidiaryRes.pCtx); + taosMemoryFreeClear(pCtx[i].subsidiaryRes.pCtx); } - tfree(pCtx); + taosMemoryFreeClear(pCtx); return NULL; } @@ -2185,12 +2232,12 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); + pRuntimeEnv->keyBuf = taosMemoryMalloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); // pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); - pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); - pRuntimeEnv->tagVal = malloc(pQueryAttr->tagLen); + pRuntimeEnv->prevRow = taosMemoryMalloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); + pRuntimeEnv->tagVal = taosMemoryMalloc(pQueryAttr->tagLen); // NOTE: pTableCheckInfo need to update the query time range and the lastKey info pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -2222,10 +2269,10 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT _clean: //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); - tfree(pRuntimeEnv->pResultRowHashTable); - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); + taosMemoryFreeClear(pRuntimeEnv->pResultRowHashTable); + taosMemoryFreeClear(pRuntimeEnv->keyBuf); + taosMemoryFreeClear(pRuntimeEnv->prevRow); + taosMemoryFreeClear(pRuntimeEnv->tagVal); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -2350,7 +2397,7 @@ static bool isCachedLastQuery(STaskAttr *pQueryAttr) { ///////////////////////////////////////////////////////////////////////////////////////////// //todo refactor : return window void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { - assert(key >= keyFirst && key <= keyLast && pInterval->sliding <= pInterval->interval); + ASSERT(key >= keyFirst && key <= keyLast); win->skey = taosTimeTruncate(key, pInterval, precision); /* @@ -2360,10 +2407,8 @@ void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t ke if (keyFirst > (INT64_MAX - pInterval->interval)) { assert(keyLast - keyFirst < pInterval->interval); win->ekey = INT64_MAX; - } else if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - win->ekey = win->skey + pInterval->interval - 1; + win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } } @@ -2761,7 +2806,7 @@ void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; - int8_t *p = calloc(numOfRows, sizeof(int8_t)); + int8_t *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); bool all = true; #if 0 if (pRuntimeEnv->pTsBuf != NULL) { @@ -2797,7 +2842,7 @@ void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo doCompactSDataBlock(pBlock, numOfRows, p); } - tfree(p); + taosMemoryFreeClear(p); } void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { @@ -2808,7 +2853,7 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, if (pRuntimeEnv->pTsBuf != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - p = calloc(numOfRows, sizeof(int8_t)); + p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); TSKEY* k = (TSKEY*) pColInfoData->pData; for (int32_t i = 0; i < numOfRows; ++i) { @@ -2844,7 +2889,7 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, } } - tfree(p); + taosMemoryFreeClear(p); } static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); @@ -3329,41 +3374,6 @@ int32_t initResultRow(SResultRow *pResultRow) { * +------------+--------------------------------------------+--------------------------------------------+ * offset[0] offset[1] offset[2] */ -void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { - SqlFunctionCtx* pCtx = pInfo->pCtx; - SSDataBlock* pDataBlock = pInfo->pRes; - int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; - SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; - - int64_t tid = 0; - pRuntimeEnv->keyBuf = realloc(pRuntimeEnv->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); - SResultRow* pRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid); - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); - - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. - */ - struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); - cleanupResultRowEntry(pEntry); - - pCtx[i].resultInfo = pEntry; - pCtx[i].pOutput = pData->pData; - pCtx[i].currentStage = stage; - assert(pCtx[i].pOutput != NULL); - - // set the timestamp output buffer for top/bottom/diff query - int32_t fid = pCtx[i].functionId; - if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { - if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } - } - - initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); -} - // TODO refactor: some function move away void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, SExecTaskInfo* pTaskInfo) { SqlFunctionCtx* pCtx = pInfo->pCtx; @@ -3376,8 +3386,6 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t SResultRow* pRow = doSetResultOutBufByKey_rv(pSup->pResultBuf, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); - struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); cleanupResultRowEntry(pEntry); @@ -3402,7 +3410,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); + char* p = taosMemoryRealloc(pColInfo->pData, newSize * pColInfo->info.bytes); if (p != NULL) { pColInfo->pData = p; @@ -3475,7 +3483,7 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (isRowEntryInitialized(pResInfo)) { + if (isRowEntryInitialized(pResInfo) || pCtx[j].functionId == -1) { continue; } @@ -3516,6 +3524,10 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCt void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t j = 0; j < numOfOutput; ++j) { + if (pCtx[j].functionId == -1) { + continue; + } + pCtx[j].fpSet.finalize(&pCtx[j]); } } @@ -3538,7 +3550,9 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD continue; } - pCtx[j].fpSet.finalize(&pCtx[j]); + if (pCtx[j].fpSet.process) { // TODO set the dummy function. + pCtx[j].fpSet.finalize(&pCtx[j]); + } if (pRow->numOfRows < pResInfo->numOfRes) { pRow->numOfRows = pResInfo->numOfRes; @@ -3584,7 +3598,7 @@ STableQueryInfo *createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow } STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { - STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); + STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(1, sizeof(STableQueryInfo)); // pTableQueryInfo->win = win; pTableQueryInfo->lastKey = win.skey; @@ -3593,7 +3607,7 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { int32_t initialSize = 16; int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); if (code != TSDB_CODE_SUCCESS) { - tfree(pTableQueryInfo); + taosMemoryFreeClear(pTableQueryInfo); return NULL; } @@ -3644,20 +3658,13 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pRes void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - SFilePage* bufPage = getBufPage(pBuf, pResult->pageId); - -// int32_t offset = 0; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { -// offset += pCtx[i].resDataInfo.bytes; continue; } - -// offset += pCtx[i].resDataInfo.bytes; - // int32_t functionId = pCtx[i].functionId; // if (functionId < 0) { // continue; @@ -3666,7 +3673,7 @@ void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, // if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput; // } - if (!pResInfo->initialized) { + if (!pResInfo->initialized && pCtx[i].functionId != -1) { pCtx[i].fpSet.init(&pCtx[i], pResInfo); } } @@ -3986,7 +3993,7 @@ static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResI static void toSDatablock(SGroupResInfo *pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - blockDataClearup(pBlock); + blockDataCleanup(pBlock); if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { return; } @@ -4026,81 +4033,6 @@ static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows, colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); } -static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) { - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SSDataBlock* pRes = pRuntimeEnv->outputBuf; - - int32_t *compSizes = NULL; - int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - - if (compressed) { - compSizes = calloc(numOfCols, sizeof(int32_t)); - } - - if (pQueryAttr->pExpr2 == NULL) { - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - if (compressed) { - compSizes[col] = compressQueryColData(pColRes, pRes->info.rows, data, compressed); - data += compSizes[col]; - *compLen += compSizes[col]; - compSizes[col] = htonl(compSizes[col]); - } else { - memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); - data += pColRes->info.bytes * pRes->info.rows; - } - } - } else { - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - if (compressed) { - compSizes[col] = htonl(compressQueryColData(pColRes, numOfRows, data, compressed)); - data += compSizes[col]; - *compLen += compSizes[col]; - compSizes[col] = htonl(compSizes[col]); - } else { - memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows); - data += pColRes->info.bytes * numOfRows; - } - } - } - - if (compressed) { - memmove(data, (char *)compSizes, numOfCols * sizeof(int32_t)); - data += numOfCols * sizeof(int32_t); - - tfree(compSizes); - } - - int32_t numOfTables = (int32_t) taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); - *(int32_t*)data = htonl(numOfTables); - data += sizeof(int32_t); - - int32_t total = 0; - STableIdInfo* item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, NULL); - - while(item) { - STableIdInfo* pDst = (STableIdInfo*)data; - pDst->uid = htobe64(item->uid); - pDst->key = htobe64(item->key); - - data += sizeof(STableIdInfo); - total++; - - //qDebug("QInfo:0x%"PRIx64" set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo->qId, item->tid, item->uid, item->key); - item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, item); - } - - //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); - - // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { -// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); - } -} - int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock *pOutput, int32_t capacity, void** p) { // for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { // SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); @@ -4121,10 +4053,10 @@ void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType e event.operatorType = operatorInfo->operatorType; if (operatorInfo->pRuntimeEnv) { - SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo; - if (pQInfo->summary.queryProfEvents) { - taosArrayPush(pQInfo->summary.queryProfEvents, &event); - } +// SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo; +// if (pQInfo->summary.queryProfEvents) { +// taosArrayPush(pQInfo->summary.queryProfEvents, &event); +// } } } @@ -4493,7 +4425,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t assert(p->numOfDownstream == 0); } - p->pDownstream = calloc(1, num * POINTER_BYTES); + p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES); if (p->pDownstream == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -4754,11 +4686,6 @@ static SSDataBlock* doTableScan(SOperatorInfo *pOperator, bool *newgroup) { STableScanInfo *pTableScanInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->_openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->pTsdbReadHandle == NULL) { return NULL; @@ -4853,7 +4780,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo *pOperator, bool* newgroup) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); int32_t len = (int32_t) tbufTell(&bw); - pColInfo->pData = malloc(len + sizeof(int32_t)); + pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); *(int32_t*) pColInfo->pData = len; memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); @@ -4869,17 +4796,27 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo *pOperator, bool* newgroup) { } static SSDataBlock* doStreamBlockScan(SOperatorInfo *pOperator, bool* newgroup) { - // NOTE: this operator never check if current status is done or not + // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; + pTaskInfo->code = pOperator->_openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; } + if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + if (pInfo->blockValid) { + pInfo->blockValid = false; // this block can only be used once. + return pInfo->pRes; + } else { + return NULL; + } + } + SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - pBlockInfo->rows = 0; + blockDataCleanup(pInfo->pRes); while (tqNextDataBlock(pInfo->readerHandle)) { pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); @@ -4924,8 +4861,8 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { @@ -4935,7 +4872,7 @@ void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; if (pMsg->contLen > 0) { - buf.pData = calloc(1, pMsg->contLen); + buf.pData = taosMemoryCalloc(1, pMsg->contLen); if (buf.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY; @@ -4952,7 +4889,7 @@ void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInfo *pTaskInfo, int32_t sourceIndex) { size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - SResFetchReq* pMsg = calloc(1, sizeof(SResFetchReq)); + SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return pTaskInfo->code; @@ -4970,9 +4907,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf pMsg->queryId = htobe64(pTaskInfo->id.queryId); // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - tfree(pMsg); + taosMemoryFreeClear(pMsg); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return pTaskInfo->code; @@ -4989,35 +4926,69 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf return TSDB_CODE_SUCCESS; } -static int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SExchangeInfo *pExchangeInfo, SSourceDataInfo* pDataInfo, int32_t numOfOutput, int64_t startTs) { - char* pData = pDataInfo->pRsp->data; - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; +// TODO if only one or two columnss required, how to extract data? +static int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t compLen, + int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { + blockDataEnsureCapacity(pRes, numOfRows); - for (int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); + if (pColList == NULL) { // data from other sources + int32_t* colLen = (int32_t*)pData; + char* pStart = pData + sizeof(int32_t) * numOfOutput; + + for (int32_t i = 0; i < numOfOutput; ++i) { + colLen[i] = htonl(colLen[i]); + ASSERT(colLen[i] > 0); + + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pColInfoData->varmeta.length = colLen[i]; + pColInfoData->varmeta.allocLen = colLen[i]; + + memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t)*numOfRows); + pStart += sizeof(int32_t)*numOfRows; + + pColInfoData->pData = taosMemoryMalloc(colLen[i]); + } else { + memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); + pStart += BitmapLen(numOfRows); + } - char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pRsp->numOfRows); - if (tmp == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + memcpy(pColInfoData->pData, pStart, colLen[i]); + pStart += colLen[i]; } + } else { // extract data according to pColList + ASSERT(numOfOutput == taosArrayGetSize(pColList)); + + // data from mnode + for(int32_t i = 0; i < numOfOutput; ++i) { - size_t len = pRsp->numOfRows * pColInfoData->info.bytes; - memcpy(tmp, pData, len); + for(int32_t j = 0; j < numOfOutput; ++j) { + int16_t colIndex = *(int16_t*) taosArrayGet(pColList, j); + if (colIndex - 1 == i) { + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, j); - pColInfoData->pData = tmp; - pData += len; + for (int32_t k = 0; k < numOfRows; ++k) { + colDataAppend(pColInfoData, k, pData, false); + pData += pColInfoData->info.bytes; + } + break; + } + } + } } - pRes->info.rows = pRsp->numOfRows; + pRes->info.rows = numOfRows; int64_t el = taosGetTimestampUs() - startTs; - pExchangeInfo->totalRows += pRsp->numOfRows; - pExchangeInfo->totalSize += pRsp->compLen; - pDataInfo->totalRows += pRsp->numOfRows; + pLoadInfo->totalRows += numOfRows; + pLoadInfo->totalSize += compLen; - pExchangeInfo->totalElapsed += el; + if (total != NULL) { + *total += numOfRows; + } + pLoadInfo->totalElapsed += el; return TSDB_CODE_SUCCESS; } @@ -5026,11 +4997,12 @@ static void* setAllSourcesCompleted(SOperatorInfo *pOperator, int64_t startTs) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int64_t el = taosGetTimestampUs() - startTs; - pExchangeInfo->totalElapsed += el; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + pLoadInfo->totalElapsed += el; size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); qDebug("%s all %"PRIzu" sources are exhausted, total rows: %"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, - pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); + pLoadInfo->totalRows, pLoadInfo->totalSize, pLoadInfo->totalElapsed/1000.0); doSetOperatorCompleted(pOperator); return NULL; @@ -5046,12 +5018,12 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo *pOperator, SEx for (int32_t i = 0; i < totalSources; ++i) { SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); - if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { + if (pDataInfo->status == DATA_EXHAUSTED) { completed += 1; continue; } - if (pDataInfo->status != EX_SOURCE_DATA_READY) { + if (pDataInfo->status != DATA_READY) { continue; } @@ -5059,17 +5031,19 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo *pOperator, SEx SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); SSDataBlock* pRes = pExchangeInfo->pResult; - + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { qDebug("%s vgId:%d, taskID:0x%" PRIx64 " index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, i + 1, pDataInfo->totalRows, - pExchangeInfo->totalRows); - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->loadInfo.totalRows); + pDataInfo->status = DATA_EXHAUSTED; completed += 1; continue; } - code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pExchangeInfo, pDataInfo, pOperator->numOfOutput, startTs); + SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; + code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, + pTableRsp->data, pTableRsp->compLen, pOperator->numOfOutput, startTs, &pDataInfo->totalRows, NULL); if (code != 0) { goto _error; } @@ -5078,17 +5052,17 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo *pOperator, SEx qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, - pDataInfo->totalRows, pExchangeInfo->totalRows, pExchangeInfo->totalSize, i + 1, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, i + 1, totalSources); - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pDataInfo->status = DATA_EXHAUSTED; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, - pExchangeInfo->totalSize); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pLoadInfo->totalRows, + pLoadInfo->totalSize); } - if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { - pDataInfo->status = EX_SOURCE_DATA_NOT_READY; + if (pDataInfo->status != DATA_EXHAUSTED) { + pDataInfo->status = DATA_NOT_READY; code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -5108,6 +5082,34 @@ _error: return NULL; } +static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo *pOperator) { + SExchangeInfo *pExchangeInfo = pOperator->info; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_RES_TO_RETURN) { + return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + // Asynchronously send all fetch requests to all sources. + for(int32_t i = 0; i < totalSources; ++i) { + int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + } + + int64_t endTs = taosGetTimestampUs(); + qDebug("%s send all fetch request to %"PRIzu" sources completed, elapsed:%"PRId64, GET_TASKID(pTaskInfo), totalSources, endTs - startTs); + + tsem_wait(&pExchangeInfo->ready); + + pOperator->status = OP_RES_TO_RETURN; + return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); +} + static int32_t prepareConcurrentlyLoad(SOperatorInfo *pOperator) { SExchangeInfo *pExchangeInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; @@ -5153,31 +5155,35 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo *pOperator) { SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, - pDataInfo->totalRows, pExchangeInfo->totalRows); + pDataInfo->totalRows, pLoadInfo->totalRows); - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pDataInfo->status = DATA_EXHAUSTED; pExchangeInfo->current += 1; continue; } SSDataBlock* pRes = pExchangeInfo->pResult; - setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pExchangeInfo, pDataInfo, pOperator->numOfOutput, startTs); + SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; + int32_t code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, + pTableRsp->data, pTableRsp->compLen, pOperator->numOfOutput, startTs, &pDataInfo->totalRows, NULL); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, - pDataInfo->totalRows, pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->current + 1, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pDataInfo->status = DATA_EXHAUSTED; pExchangeInfo->current += 1; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->totalSize); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pLoadInfo->totalRows, pLoadInfo->totalSize); } return pExchangeInfo->pResult; @@ -5213,23 +5219,26 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo *pOperator, bool* newgroup) { } size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pOperator->status == OP_EXEC_DONE) { qDebug("%s all %"PRIzu" source(s) are exhausted, total rows:%"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, - pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); + pLoadInfo->totalRows, pLoadInfo->totalSize, pLoadInfo->totalElapsed/1000.0); return NULL; } *newgroup = false; + if (pExchangeInfo->seqLoadData) { return seqLoadRemoteData(pOperator); } else { - return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + return concurrentlyLoadRemoteData(pOperator); } #if 0 _error: - tfree(pMsg); - tfree(pMsgSendInfo); + taosMemoryFreeClear(pMsg); + taosMemoryFreeClear(pMsgSendInfo); terrno = pTaskInfo->code; return NULL; @@ -5259,11 +5268,14 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { } SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - goto _error; + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; } size_t numOfSources = LIST_LENGTH(pSources); @@ -5287,7 +5299,6 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock pInfo->pResult = pBlock; pInfo->seqLoadData = true; - pInfo->seqLoadData = true; // sequentially load data from the source node tsem_init(&pInfo->ready, 0, 0); pOperator->name = "ExchangeOperator"; @@ -5331,14 +5342,14 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock destroyExchangeOperatorInfo(pInfo, numOfSources); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { - SSDataBlock* pResBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pResBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pResBlock == NULL) { return NULL; } @@ -5366,11 +5377,11 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -5388,16 +5399,14 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->_openFn = operatorDummyOpenFn; pOperator->getNextFn = doTableScan; - pOperator->closeFn = operatorDummyCloseFn; pOperator->pTaskInfo = pTaskInfo; return pOperator; } SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->times = 1; @@ -5407,7 +5416,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim pInfo->prevGroupId = -1; pRuntimeEnv->enableGroupData = true; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableSeqScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; pOperator->blockingOptr = false; @@ -5421,7 +5430,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim } SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); @@ -5432,7 +5441,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt infoData.info.colId = 0; taosArrayPush(pInfo->block.pDataBlock, &infoData); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; pOperator->blockingOptr = false; @@ -5445,11 +5454,11 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt } SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { - SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -5458,8 +5467,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); if (code != 0) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); return NULL; } @@ -5475,23 +5484,125 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* pOperator->_openFn = operatorDummyOpenFn; pOperator->getNextFn = doStreamBlockScan; pOperator->closeFn = operatorDummyCloseFn; - pOperator->pTaskInfo = pTaskInfo; return pOperator; } - static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t code) { - SSourceDataInfo* pSourceDataInfo = (SSourceDataInfo*) param; - pSourceDataInfo->pRsp = pMsg->pData; + SOperatorInfo* operator = (SOperatorInfo *)param; + SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo *)operator->info; + if (TSDB_CODE_SUCCESS == code) { + pScanResInfo->pRsp = pMsg->pData; + + SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->handle = htobe64(pRsp->handle); + pRsp->compLen = htonl(pRsp->compLen); + } else { + operator->pTaskInfo->code = code; + } - SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); + tsem_post(&pScanResInfo->ready); +} - pSourceDataInfo->status = EX_SOURCE_DATA_READY; - tsem_post(&pSourceDataInfo->pEx->ready); + +static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { + if (pInfo->pCondition == NULL) { + return pInfo->pRes->info.rows == 0? NULL:pInfo->pRes; + } + + SFilterInfo* filter = NULL; + int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); + + SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m1); + + int8_t* rowRes = NULL; + bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); + + SSDataBlock* px = createOneDataBlock(pInfo->pRes); + blockDataEnsureCapacity(px, pInfo->pRes->info.rows); + + //TODO refactor + int32_t numOfRow = 0; + for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { + SColumnInfoData* pDest = taosArrayGet(px->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); + + numOfRow = 0; + for (int32_t j = 0; j < pInfo->pRes->info.rows; ++j) { + if (rowRes[j] == 0) { + continue; + } + + colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); + numOfRow += 1; + } + } + + px->info.rows = numOfRow; + pInfo->pRes = px; + + return pInfo->pRes->info.rows == 0? NULL:pInfo->pRes; +} + +EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { + int32_t code = TSDB_CODE_SUCCESS; + ENodeType nType = nodeType(pNode); + + switch (nType) { + case QUERY_NODE_OPERATOR: { + SOperatorNode *node = (SOperatorNode *)pNode; + + if (OP_TYPE_EQUAL == node->opType) { + *(int32_t *)pContext = 1; + return DEAL_RES_CONTINUE; + } + + *(int32_t *)pContext = 0; + + return DEAL_RES_IGNORE_CHILD; + } + case QUERY_NODE_COLUMN: { + if (1 != *(int32_t *)pContext) { + return DEAL_RES_CONTINUE; + } + + SColumnNode *node = (SColumnNode *)pNode; + if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) { + *(int32_t *)pContext = 2; + return DEAL_RES_CONTINUE; + } + + *(int32_t *)pContext = 0; + return DEAL_RES_CONTINUE; + } + case QUERY_NODE_VALUE: { + if (2 != *(int32_t *)pContext) { + return DEAL_RES_CONTINUE; + } + + SValueNode *node = (SValueNode *)pNode; + char *dbName = nodesGetValueFromNode(node); + strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); + *((char *)pContext + varDataLen(dbName)) = 0; + return DEAL_RES_ERROR; // stop walk + } + default: + break; + } + + return DEAL_RES_CONTINUE; +} + + +void getDBNameFromCondition(SNode *pCondition, char *dbName) { + if (NULL == pCondition) { + return; + } + + nodesWalkNode(pCondition, getDBNameFromConditionWalker, dbName); } static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { @@ -5505,88 +5616,193 @@ static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); } - SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, 0); + blockDataCleanup(pInfo->pRes); + + int32_t tableNameSlotId = 1; + SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); char * name = NULL; int32_t numOfRows = 0; + + char n[TSDB_TABLE_NAME_LEN] = {0}; while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) { - colDataAppend(pTableNameCol, numOfRows, name, false); + STR_TO_VARSTR(n, name); + colDataAppend(pTableNameCol, numOfRows, n, false); numOfRows += 1; if (numOfRows >= pInfo->capacity) { break; } + + for(int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { + if (i == tableNameSlotId) { + continue; + } + + SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); + int64_t tmp = 0; + char t[10] = {0}; + STR_TO_VARSTR(t, "_"); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + colDataAppend(pColInfoData, numOfRows, t, false); + } else { + colDataAppend(pColInfoData, numOfRows, (char*) &tmp, false); + } + } } - pInfo->totalRows += numOfRows; + pInfo->loadInfo.totalRows += numOfRows; pInfo->pRes->info.rows = numOfRows; // pInfo->elapsedTime; // pInfo->totalBytes; return (pInfo->pRes->info.rows == 0)? NULL:pInfo->pRes; } else { // load the meta from mnode of the given epset - if (pInfo->pReq == NULL) { - pInfo->pReq = calloc(1, sizeof(SRetrieveTableReq)); - if (pInfo->pReq == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } + int64_t startTs = taosGetTimestampUs(); - pInfo->pReq->type = pInfo->type; + pInfo->req.type = pInfo->type; + strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + if (pInfo->showRewrite) { + char dbName[TSDB_DB_NAME_LEN] = {0}; + getDBNameFromCondition(pInfo->pCondition, dbName); + sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); } + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); + char* buf1 = taosMemoryCalloc(1, contLen); + tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); + // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } - pMsgSendInfo->param = NULL; - pMsgSendInfo->msgInfo.pData = pInfo->pReq; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); + pMsgSendInfo->param = pOperator; + pMsgSendInfo->msgInfo.pData = buf1; + pMsgSendInfo->msgInfo.len = contLen; pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; - pMsgSendInfo->fp = loadRemoteDataCallback; + pMsgSendInfo->fp = loadSysTableContentCb; int64_t transporterId = 0; int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pInfo->ready); - // handle the response and return to the caller + + if (pTaskInfo->code) { + return NULL; + } + + SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; + pInfo->req.showId = pRsp->handle; + + if (pRsp->numOfRows == 0) { +// qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", +// GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, +// pDataInfo->totalRows, pExchangeInfo->totalRows); + return NULL; + } + + SRetrieveMetaTableRsp* pTableRsp = pInfo->pRsp; + setSDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pTableRsp->numOfRows, + pTableRsp->data, pTableRsp->compLen, pOperator->numOfOutput, startTs, NULL, pInfo->scanCols); + + return doFilterResult(pInfo); } return NULL; } -SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, const SArray* pExprInfo, const SSchema* pSchema, - int32_t tableType, SEpSet epset, SExecTaskInfo* pTaskInfo) { - SSysTableScanInfo* pInfo = calloc(1, sizeof(SSysTableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); +SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, + SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId) { + SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } - // todo: create the schema of result data block - pInfo->capacity = 4096; - pInfo->type = tableType; + pInfo->accountId = accountId; + pInfo->showRewrite = showRewrite; + pInfo->pRes = pResBlock; + pInfo->capacity = 4096; + pInfo->pCondition = pCondition; + pInfo->scanCols = colList; + + // TODO remove it + int32_t tableType = 0; + const char* name = tNameGetTableName(pName); + if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_DB; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_USER; + } else if (strncasecmp(name, TSDB_INS_TABLE_DNODES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_DNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_MNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_MODULE; + } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_QNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_FUNC; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, tListLen(pName->tname)) == 0) { +// tableType = TSDB_MGMT_TABLE_INDEX; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_STB; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_STREAMS; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_TABLE; + } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, tListLen(pName->tname)) == 0) { + tableType = TSDB_MGMT_TABLE_VGROUP; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, tListLen(pName->tname)) == 0) { +// tableType = TSDB_MGMT_TABLE_DIST; + } else { + ASSERT(0); + } + + tNameAssign(&pInfo->name, pName); + pInfo->type = tableType; if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { pInfo->readHandle = pSysTableReadHandle; blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); } else { tsem_init(&pInfo->ready, 0, 0); pInfo->epSet = epset; + +#if 1 + { // todo refactor + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "DB-META"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = qProcessFetchRsp; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)"root"; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; + + pInfo->pTransporter = rpcOpen(&rpcInit); + if (pInfo->pTransporter == NULL) { + return NULL; // todo + } + } +#endif } - pInfo->readHandle = pSysTableReadHandle; pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; + pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->numOfOutput = taosArrayGetSize(pExprInfo); + pOperator->numOfOutput = pResBlock->info.numOfCols; pOperator->getNextFn = doSysTableScan; pOperator->closeFn = destroySysTableScannerOperatorInfo; pOperator->pTaskInfo = pTaskInfo; @@ -5594,64 +5810,6 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, const S return pOperator; } -void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { - assert(pTableScanInfo != NULL && pDownstream != NULL); - - pTableScanInfo->pExpr = pDownstream->pExpr; // TODO refactor to use colId instead of pExpr - pTableScanInfo->numOfOutput = pDownstream->numOfOutput; -#if 0 - if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) { - SAggOperatorInfo* pAggInfo = pDownstream->info; - - pTableScanInfo->pCtx = pAggInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_TimeWindow || pDownstream->operatorType == OP_AllTimeWindow) { - STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info; - - pTableScanInfo->pCtx = pIntervalInfo->pCtx; - pTableScanInfo->pResultRowInfo = &pIntervalInfo->resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pIntervalInfo->rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_Groupby) { - SGroupbyOperatorInfo *pGroupbyInfo = pDownstream->info; - - pTableScanInfo->pCtx = pGroupbyInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_MultiTableTimeInterval || pDownstream->operatorType == OP_AllMultiTableTimeInterval) { - STableIntervalOperatorInfo *pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_Project) { - SProjectOperatorInfo *pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_SessionWindow) { - SSessionAggOperatorInfo* pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_StateWindow) { - SStateWindowOperatorInfo* pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else { - assert(0); - } -#endif - -} - SArray* getOrderCheckColumns(STaskAttr* pQuery) { int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0: taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); @@ -5746,7 +5904,7 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; taosArrayDestroy(pInfo->orderColumnList); pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->prevRow); + taosMemoryFreeClear(pInfo->prevRow); } static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { @@ -5755,7 +5913,7 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { *dst = *src; dst->pExpr = exprdup(src->pExpr); - dst->base.pParam = calloc(src->base.numOfParams, sizeof(SColumn)); + dst->base.pParam = taosMemoryCalloc(src->base.numOfParams, sizeof(SColumn)); memcpy(dst->base.pParam, src->base.pParam, sizeof(SColumn) * src->base.numOfParams); // memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); @@ -5767,7 +5925,7 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { static SExprInfo* exprArrayDup(SArray* pExprList) { size_t numOfOutput = taosArrayGetSize(pExprList); - SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); + SExprInfo* p = taosMemoryCalloc(numOfOutput, sizeof(SExprInfo)); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExpr = taosArrayGetP(pExprList, i); assignExprInfo(&p[i], pExpr); @@ -5794,7 +5952,7 @@ static void appendOneRowToDataBlock(SSDataBlock *pBlock, STupleHandle* pTupleHan } static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, bool hasVarCol, int32_t capacity) { - blockDataClearup(pDataBlock); + blockDataCleanup(pDataBlock); while(1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); @@ -5950,7 +6108,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { while(1) { - blockDataClearup(pDataBlock); + blockDataCleanup(pDataBlock); while (1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); if (pTupleHandle == NULL) { @@ -6001,11 +6159,11 @@ static SSDataBlock* doSortedMerge(SOperatorInfo *pOperator, bool* newgroup) { pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, p, pInfo->binfo.pRes->info.numOfCols, "GET_TASKID(pTaskInfo)"); - tfree(p); + taosMemoryFreeClear(p); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) { - SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); ps->param = pOperator->pDownstream[i]; tsortAddSource(pInfo->pSortHandle, ps); } @@ -6071,7 +6229,7 @@ static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGr ASSERT(taosArrayGetSize(pGroupInfo) == taosArrayGetSize(plist)); - pInfo->groupVal = calloc(1, (POINTER_BYTES * numOfGroupCol + len)); + pInfo->groupVal = taosMemoryCalloc(1, (POINTER_BYTES * numOfGroupCol + len)); if (pInfo->groupVal == NULL) { taosArrayDestroy(plist); return TSDB_CODE_OUT_OF_MEMORY; @@ -6091,8 +6249,8 @@ static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGr } SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo) { - SSortedMergeOperatorInfo* pInfo = calloc(1, sizeof(SSortedMergeOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SSortedMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortedMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -6147,8 +6305,8 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t destroySortedMergeOperatorInfo(pInfo, num); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -6169,10 +6327,10 @@ static SSDataBlock* doSort(SOperatorInfo *pOperator, bool* newgroup) { pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage, p, pInfo->pDataBlock->info.numOfCols, "GET_TASKID(pTaskInfo)"); - tfree(p); + taosMemoryFreeClear(p); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); ps->param = pOperator; tsortAddSource(pInfo->pSortHandle, ps); @@ -6187,11 +6345,11 @@ static SSDataBlock* doSort(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo) { - SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOrderOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SOrderOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -6210,9 +6368,9 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx } if (pInfo->orderInfo == NULL || pInfo->pDataBlock == NULL) { - tfree(pOperator); + taosMemoryFreeClear(pOperator); destroyOrderOperatorInfo(pInfo, numOfCols); - tfree(pInfo); + taosMemoryFreeClear(pInfo); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -6293,6 +6451,111 @@ static SSDataBlock* getAggregateResult(SOperatorInfo *pOperator, bool* newgroup) return (blockDataGetNumOfRows(pInfo->pRes) != 0)? pInfo->pRes:NULL; } +static void aggEncodeResultRow(SOperatorInfo* pOperator, char **result, int32_t *length) { + SAggOperatorInfo *pAggInfo = pOperator->info; + SAggSupporter *pSup = &pAggInfo->aggSup; + + int32_t size = taosHashGetSize(pSup->pResultRowHashTable); + size_t keyLen = POINTER_BYTES; // estimate the key length + int32_t totalSize = sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); + *result = taosMemoryCalloc(1, totalSize); + if(*result == NULL){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + *(int32_t*)(*result) = size; + int32_t offset = sizeof(int32_t); + void *pIter = taosHashIterate(pSup->pResultRowHashTable, NULL); + while (pIter) { + void *key = taosHashGetKey(pIter, &keyLen); + SResultRow **p1 = (SResultRow **)pIter; + + // recalculate the result size + int32_t realTotalSize = offset + sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize; + if (realTotalSize > totalSize){ + char *tmp = taosMemoryRealloc(*result, realTotalSize); + if (tmp == NULL){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(*result); + *result = NULL; + return; + }else{ + *result = tmp; + } + } + // save key + *(int32_t*)(*result + offset) = keyLen; + offset += sizeof(int32_t); + memcpy(*result + offset, key, keyLen); + offset += keyLen; + + // save value + *(int32_t*)(*result + offset) = pSup->resultRowSize; + offset += sizeof(int32_t); + memcpy(*result + offset, *p1, pSup->resultRowSize); + offset += pSup->resultRowSize; + + pIter = taosHashIterate(pSup->pResultRowHashTable, pIter); + } + + if(length) { + *length = offset; + } + return; +} + +static bool aggDecodeResultRow(SOperatorInfo* pOperator, char *result, int32_t length) { + if (!result || length <= 0){ + return false; + } + + SAggOperatorInfo *pAggInfo = pOperator->info; + SAggSupporter *pSup = &pAggInfo->aggSup; + SOptrBasicInfo *pInfo = &pAggInfo->binfo; + + // int32_t size = taosHashGetSize(pSup->pResultRowHashTable); + int32_t count = *(int32_t*)(result); + + int32_t offset = sizeof(int32_t); + while(count-- > 0 && length > offset){ + int32_t keyLen = *(int32_t*)(result + offset); + offset += sizeof(int32_t); + + uint64_t tableGroupId = *(uint64_t *)(result + offset); + SResultRow *resultRow = getNewResultRow_rv(pSup->pResultBuf, tableGroupId, pSup->resultRowSize); + if (!resultRow){ + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return false; + } + // add a new result set for a new group + taosHashPut(pSup->pResultRowHashTable, result + offset, keyLen, &resultRow, POINTER_BYTES); + + offset += keyLen; + int32_t valueLen = *(int32_t*)(result + offset); + if (valueLen != pSup->resultRowSize){ + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return false; + } + offset += sizeof(int32_t); + int32_t pageId = resultRow->pageId; + int32_t pOffset = resultRow->offset; + memcpy(resultRow, result + offset, valueLen); + resultRow->pageId = pageId; + resultRow->offset = pOffset; + offset += valueLen; + + initResultRow(resultRow); + + pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] = (SResultRowPosition) {.pageId = resultRow->pageId, .offset = resultRow->offset}; + } + + if (offset != length){ + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return false; + } + return true; +} + static SSDataBlock* doMultiTableAggregate(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -6366,7 +6629,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo *pOperator, bool* newgroup) SOptrBasicInfo *pInfo = &pProjectInfo->binfo; SSDataBlock* pRes = pInfo->pRes; - blockDataClearup(pRes); + blockDataCleanup(pRes); if (pProjectInfo->existDataBlock) { // TODO refactor // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; @@ -6381,12 +6644,12 @@ static SSDataBlock* doProjectOperation(SOperatorInfo *pOperator, bool* newgroup) // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); - updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows); - projectApplyFunctions(pInfo->pCtx, pOperator->numOfOutput); + blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); + projectApplyFunctions(pInfo->pRes, pInfo->pCtx, pOperator->numOfOutput); pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput, NULL); - if (pRes->info.rows >= pProjectInfo->threshold) { + if (pRes->info.rows >= pProjectInfo->binfo.capacity*0.8) { copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput); return pRes; @@ -6416,15 +6679,11 @@ static SSDataBlock* doProjectOperation(SOperatorInfo *pOperator, bool* newgroup) pProjectInfo->existDataBlock = pBlock; break; } else { // init output buffer for a new group data -// for (int32_t j = 0; j < pOperator->numOfOutput; ++j) { -// aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]); -// } initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfOutput); } } // todo dynamic set tags - // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // if (pTableQueryInfo != NULL) { // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); @@ -6434,15 +6693,13 @@ static SSDataBlock* doProjectOperation(SOperatorInfo *pOperator, bool* newgroup) setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows); - projectApplyFunctions(pInfo->pCtx, pOperator->numOfOutput); -// pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput, pRes); + projectApplyFunctions(pInfo->pRes, pInfo->pCtx, pOperator->numOfOutput); if (pRes->info.rows >= pProjectInfo->threshold) { break; } } copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); -// resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput); return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } @@ -6662,10 +6919,6 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo *pOperator, bool* newgroup if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); } - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - return pIntervalInfo->binfo.pRes; } @@ -6758,8 +7011,8 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo *pOperator, bool* newgr pOperator->status = OP_EXEC_DONE; } - SQInfo* pQInfo = pRuntimeEnv->qinfo; - pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); +// SQInfo* pQInfo = pRuntimeEnv->qinfo; +// pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); return pIntervalInfo->binfo.pRes; } @@ -6779,7 +7032,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI TSKEY* tsList = (TSKEY*)pTsColInfoData->pData; if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { pInfo->reptScan = true; - tfree(pInfo->prevData); + taosMemoryFreeClear(pInfo->prevData); } pInfo->numOfRows = 0; @@ -6789,7 +7042,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI continue; } if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); + pInfo->prevData = taosMemoryMalloc(bytes); memcpy(pInfo->prevData, val, bytes); pInfo->numOfRows = 1; pInfo->curWindow.skey = tsList[j]; @@ -6950,18 +7203,15 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo *pOperator, bool* newgrou } SGroupbyOperatorInfo *pInfo = pOperator->info; - - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { -// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset); + if (pInfo->binfo.pRes->info.rows == 0|| !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pInfo->binfo.pRes; } + int32_t order = TSDB_ORDER_ASC; SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { @@ -6973,84 +7223,80 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo *pOperator, bool* newgrou } // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQueryAttr->order.order); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order); // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); - if (pInfo->colIndex == -1) { - pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); - } - - doHashGroupbyAgg(pOperator, pInfo, pBlock); + doHashGroupbyAgg(pOperator, pBlock); } pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->binfo.resultRowInfo); -// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); - if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows - finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput); - } else { - updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - } - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); - if (!pRuntimeEnv->pQueryAttr->stableQuery) { - sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - } + finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); +// if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows +// finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput); +// } else { +// updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); +// } -// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); + initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } return pInfo->binfo.pRes; } -static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { +static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SResultInfo* pResultInfo, bool* newgroup, SExecTaskInfo* pTaskInfo) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; + + int64_t ekey = Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED)? pTaskInfo->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pResultInfo->capacity, pInfo->p); pInfo->existNewGroupBlock = NULL; *newgroup = true; } -static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, STaskRuntimeEnv *pRuntimeEnv, bool *newgroup) { +static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, SResultInfo *pResultInfo, bool *newgroup, SExecTaskInfo* pTaskInfo) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { *newgroup = false; - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult)) { + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pResultInfo->capacity, pInfo->p); + if (pInfo->pRes->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult)) { return; } } // handle the cached new group data block if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); + doHandleRemainBlockForNewGroupImpl(pInfo, pResultInfo, newgroup, pTaskInfo); } } static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { SFillOperatorInfo *pInfo = pOperator->info; - pInfo->pRes->info.rows = 0; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SResultInfo* pResultInfo = &pOperator->resultInfo; + blockDataCleanup(pInfo->pRes); if (pOperator->status == OP_EXEC_DONE) { return NULL; } - STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { + doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); + if (pInfo->pRes->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { return pInfo->pRes; } + SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->pDownstream[0]->getNextFn(pOperator->pDownstream[0], newgroup); - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pDownstream->getNextFn(pDownstream, newgroup); + publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { assert(pBlock != NULL); @@ -7062,7 +7308,7 @@ static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { // Fill the previous group data block, before handle the data block of new group. // Close the fill operation for previous group data block - taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); + taosFillSetStartInfo(pInfo->pFillInfo, 0, pTaskInfo->window.ekey); } else { if (pBlock == NULL) { if (pInfo->totalInputRows == 0) { @@ -7070,7 +7316,7 @@ static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { return NULL; } - taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); + taosFillSetStartInfo(pInfo->pFillInfo, 0, pTaskInfo->window.ekey); } else { pInfo->totalInputRows += pBlock->info.rows; taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey); @@ -7078,25 +7324,24 @@ static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { } } - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pInfo->capacity, pInfo->p); // current group has no more result to return if (pInfo->pRes->info.rows > 0) { // 1. The result in current group not reach the threshold of output result, continue // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL || (!pInfo->multigroupResult)) { + if (pInfo->pRes->info.rows > pResultInfo->threshold || pBlock == NULL || (!pInfo->multigroupResult)) { return pInfo->pRes; } - doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL) { + doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); + if (pInfo->pRes->info.rows > pOperator->resultInfo.threshold || pBlock == NULL) { return pInfo->pRes; } } else if (pInfo->existNewGroupBlock) { // try next group assert(pBlock != NULL); - doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); - - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold) { + doHandleRemainBlockForNewGroupImpl(pInfo, pResultInfo, newgroup, pTaskInfo); + if (pInfo->pRes->info.rows > pResultInfo->threshold) { return pInfo->pRes; } } else { @@ -7131,19 +7376,19 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { destroyOperatorInfo(pOperator->pDownstream[i]); } - tfree(pOperator->pDownstream); + taosMemoryFreeClear(pOperator->pDownstream); pOperator->numOfDownstream = 0; } - tfree(pOperator->info); - tfree(pOperator); + taosMemoryFreeClear(pOperator->info); + taosMemoryFreeClear(pOperator); } int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput, const char* pKey) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); + pAggSup->keyBuf = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); @@ -7162,7 +7407,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t n } static void cleanupAggSup(SAggSupporter* pAggSup) { - tfree(pAggSup->keyBuf); + taosMemoryFreeClear(pAggSup->keyBuf); taosHashCleanup(pAggSup->pResultRowHashTable); taosHashCleanup(pAggSup->pResultRowListSet); taosArrayDestroy(pAggSup->pResultRowArrayList); @@ -7179,7 +7424,7 @@ static int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, S } static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInfo) { - STableQueryInfo* pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pTableQueryInfo == NULL) { return NULL; } @@ -7204,8 +7449,8 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7232,6 +7477,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->_openFn = doOpenAggregateOptr; pOperator->getNextFn = getAggregateResult; pOperator->closeFn = destroyAggOperatorInfo; + pOperator->encodeResultRow = aggEncodeResultRow; + pOperator->decodeResultRow = aggDecodeResultRow; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -7241,8 +7488,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* return pOperator; _error: destroyAggOperatorInfo(pInfo, numOfCols); - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -7251,7 +7498,7 @@ static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { assert(pInfo != NULL); destroySqlFunctionCtx(pInfo->pCtx, numOfOutput); - tfree(pInfo->rowCellInfoOffset); + taosMemoryFreeClear(pInfo->rowCellInfoOffset); cleanupResultRowInfo(&pInfo->resultRowInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); @@ -7265,7 +7512,7 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); + taosMemoryFreeClear(pInfo->prevData); } void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { @@ -7288,26 +7535,28 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->p); + taosMemoryFreeClear(pInfo->p); } void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); + taosMemoryFreeClear(pInfo->keyBuf); + taosArrayDestroy(pInfo->pGroupCols); + taosArrayDestroy(pInfo->pGroupColVals); } -void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { STagScanInfo* pInfo = (STagScanInfo*) param; pInfo->pRes = blockDataDestroy(pInfo->pRes); } -void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { +static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param; pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); @@ -7322,7 +7571,7 @@ static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param; taosHashCleanup(pInfo->pSet); - tfree(pInfo->buf); + taosMemoryFreeClear(pInfo->buf); taosArrayDestroy(pInfo->pDistinctDataInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); } @@ -7349,7 +7598,7 @@ void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { } SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); int32_t numOfRows = 1; int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); @@ -7361,7 +7610,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprI size_t tableGroup = taosArrayGetSize(pTableGroupInfo->pGroupList); initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableAggregate"; // pOperator->operatorType = OP_MultiTableAggregate; pOperator->blockingOptr = true; @@ -7385,8 +7634,8 @@ _error: } SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { - SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7407,14 +7656,13 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p pOperator->info = pInfo; pOperator->pExpr = pExprInfo; pOperator->numOfOutput = num; - pOperator->_openFn = operatorDummyOpenFn; pOperator->getNextFn = doProjectOperation; pOperator->closeFn = destroyProjectOperatorInfo; pOperator->pTaskInfo = pTaskInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_OUT_OF_MEMORY) { + if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -7427,7 +7675,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) { #if 0 - SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); + SColumnInfo* pCols = taosMemoryCalloc(numOfOutput, sizeof(SColumnInfo)); int32_t numOfFilter = 0; for(int32_t i = 0; i < numOfOutput; ++i) { @@ -7441,7 +7689,7 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; if (pCols[i].flist.numOfFilters != 0) { - pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); + pCols[i].flist.filterInfo = taosMemoryCalloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); } else { // avoid runtime error @@ -7458,10 +7706,9 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 return 0; } -SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfDownstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { - ASSERT(numOfDownstream == 1); - SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); +SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { + SLimitOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SLimitOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7481,22 +7728,22 @@ SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfD return pOperator; _error: - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } pInfo->order = TSDB_ORDER_ASC; - pInfo->precision = TSDB_TIME_PRECISION_MICRO; + pInfo->precision = TSDB_TIME_PRECISION_MILLI; pInfo->win = pTaskInfo->window; pInfo->interval = *pInterval; @@ -7517,7 +7764,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->blockingOptr = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; @@ -7534,20 +7780,20 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* _error: destroyIntervalOperatorInfo(pInfo, numOfCols); - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; } SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllTimeIntervalAggOperator"; // pOperator->operatorType = OP_AllTimeWindow; @@ -7565,14 +7811,14 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S } SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); + SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); pInfo->colIndex = -1; pInfo->reptScan = false; pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "StateWindowOperator"; // pOperator->operatorType = OP_StateWindow; pOperator->blockingOptr = true; @@ -7589,8 +7835,8 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper } SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { - SSessionAggOperatorInfo* pInfo = calloc(1, sizeof(SSessionAggOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7611,7 +7857,6 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; pOperator->getNextFn = doSessionWindowAgg; pOperator->closeFn = destroySWindowOperatorInfo; @@ -7625,20 +7870,20 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo destroySWindowOperatorInfo(pInfo, numOfCols); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; } SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableTimeIntervalOperator"; // pOperator->operatorType = OP_MultiTableTimeInterval; pOperator->blockingOptr = true; @@ -7656,13 +7901,13 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim } SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllMultiTableTimeIntervalOperator"; // pOperator->operatorType = OP_AllMultiTableTimeInterval; pOperator->blockingOptr = true; @@ -7680,83 +7925,146 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun return pOperator; } -SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); - pInfo->colIndex = -1; // group by column index +static int32_t initGroupOptrInfo(SGroupbyOperatorInfo *pInfo, SArray* pGroupColList) { + pInfo->pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); + if (pInfo->pGroupColVals == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for(int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pGroupColList, i); + pInfo->groupKeyLen += pCol->bytes; - pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + struct SGroupKeys key = {0}; + key.bytes = pCol->bytes; + key.type = pCol->type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pCol->bytes); + if (key.pData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + taosArrayPush(pInfo->pGroupColVals, &key); + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + pInfo->keyBuf = taosMemoryCalloc(1, pInfo->groupKeyLen + nullFlagSize); + + if (pInfo->keyBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * - (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); + return TSDB_CODE_SUCCESS; +} - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, + SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { + SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->pGroupCols = pGroupColList; + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pTaskInfo->id.str); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + int32_t code = initGroupOptrInfo(pInfo, pGroupColList); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pOperator->name = "GroupbyAggOperator"; pOperator->blockingOptr = true; pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Groupby; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->getNextFn = hashGroupbyAggregate; - pOperator->closeFn = destroyGroupbyOperatorInfo; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = hashGroupbyAggregate; + pOperator->closeFn = destroyGroupbyOperatorInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; } -SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { - SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - pInfo->multigroupResult = multigroupResult; +static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, int64_t* fillVal, + STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { + struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, (int64_t*)fillVal); - { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); - STimeWindow w = TSWINDOW_INITIALIZER; + TSKEY sk = TMIN(win.skey, win.ekey); + TSKEY ek = TMAX(win.skey, win.ekey); - TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); + // TODO set correct time precision + STimeWindow w = TSWINDOW_INITIALIZER; + getAlignQueryTimeWindow(pInterval, TSDB_TIME_PRECISION_MILLI, win.skey, sk, ek, &w); + + int32_t order = TSDB_ORDER_ASC; + pInfo->pFillInfo = + taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval->sliding, + pInterval->slidingUnit, (int8_t)pInterval->precision, fillType, pColInfo, id); - pInfo->pFillInfo = - taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, - pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, - (int8_t)pQueryAttr->precision, pQueryAttr->fillType, pColInfo, pRuntimeEnv->qinfo); + pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); - pInfo->p = calloc(numOfOutput, POINTER_BYTES); + if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } else { + return TSDB_CODE_SUCCESS; } +} - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); +SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, + int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo) { + SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + + pInfo->pRes = pResBlock; + pInfo->multigroupResult = multigroupResult; + pInfo->intervalInfo = *pInterval; + + SResultInfo* pResultInfo = &pOperator->resultInfo; + int32_t code = initFillInfo(pInfo, pExpr, numOfCols, (int64_t*) fillVal, pTaskInfo->window, pResultInfo->capacity, pTaskInfo->id.str, pInterval, fillType); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pOperator->name = "FillOperator"; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Fill; pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; + pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->getNextFn = doFill; - pOperator->closeFn = destroySFillOperatorInfo; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doFill; + pOperator->pTaskInfo = pTaskInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + pOperator->closeFn = destroySFillOperatorInfo; + + code = appendDownstream(pOperator, &downstream, 1); return pOperator; + + _error: + taosMemoryFreeClear(pOperator); + taosMemoryFreeClear(pInfo); + return NULL; } SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { - SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); + SSLimitOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSLimitOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); // pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); // pInfo->slimit = pQueryAttr->slimit; // pInfo->limit = pQueryAttr->limit; -// pInfo->capacity = pRuntimeEnv->resultInfo.capacity; +// pInfo->capacity = pResultInfo->capacity; // pInfo->threshold = (int64_t)(pInfo->capacity * 0.8); // pInfo->currentOffset = pQueryAttr->limit.offset; // pInfo->currentGroupOffset = pQueryAttr->slimit.offset; @@ -7769,7 +8077,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI } int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + pInfo->prevRow = taosMemoryCalloc(1, (POINTER_BYTES * numOfCols + len)); int32_t offset = POINTER_BYTES * numOfCols; for(int32_t i = 0; i < numOfCols; ++i) { @@ -7779,9 +8087,8 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI offset += pExpr[index->colIndex].base.resSchema.bytes; } - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pOperator->resultInfo.capacity); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SLimitOperator"; // pOperator->operatorType = OP_SLimit; @@ -7789,9 +8096,10 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->status = OP_NOT_OPENED; // pOperator->exec = doSLimit; pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->closeFn = destroySlimitOperatorInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7803,7 +8111,7 @@ static SSDataBlock* doTagScan(SOperatorInfo *pOperator, bool* newgroup) { } STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; + int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; STagScanInfo *pInfo = pOperator->info; SSDataBlock *pRes = pInfo->pRes; @@ -7925,12 +8233,11 @@ static SSDataBlock* doTagScan(SOperatorInfo *pOperator, bool* newgroup) { return (pRes->info.rows == 0)? NULL:pInfo->pRes; #endif - return 0; } SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { - STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); +// pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); assert(numOfGroup == 0 || numOfGroup == 1); @@ -7938,7 +8245,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; pInfo->curPos = 0; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "SeqTableTagScan"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; pOperator->blockingOptr = false; @@ -7972,7 +8279,7 @@ static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* p } } pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); - pInfo->buf = calloc(1, pInfo->totalBytes); + pInfo->buf = taosMemoryCalloc(1, pInfo->totalBytes); return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; } @@ -8030,7 +8337,7 @@ static SSDataBlock* hashDistinct(SOperatorInfo *pOperator, bool* newgroup) { for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); - char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); + char* tmp = taosMemoryRealloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); if (tmp == NULL) { return NULL; } else { @@ -8066,7 +8373,7 @@ static SSDataBlock* hashDistinct(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); + SDistinctOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDistinctOperatorInfo)); pInfo->totalBytes = 0; pInfo->buf = NULL; pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold @@ -8076,7 +8383,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "DistinctOperator"; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; @@ -8129,37 +8436,6 @@ bool validateExprColumnInfo(SQueriedTableInfo *pTableInfo, SExprBasicInfo *pExpr return j != INT32_MIN; } -static bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SExprBasicInfo** pExpr, int32_t numOfOutput, - SColumnInfo* pTagCols, void* pMsg) { - int32_t numOfTotal = pTableInfo->numOfCols + pTableInfo->numOfTags; - if (pTableInfo->numOfCols < 0 || pTableInfo->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { - //qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pMsg, pTableInfo->numOfCols, pTableInfo->numOfTags); - return false; - } - - if (numOfTotal == 0) { // table total columns are not required. -// for(int32_t i = 0; i < numOfOutput; ++i) { -// SExprBasicInfo* p = pExpr[i]; -// if ((p->functionId == FUNCTION_TAGPRJ) || -// (p->functionId == FUNCTION_TID_TAG && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || -// (p->functionId == FUNCTION_COUNT && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || -// (p->functionId == FUNCTION_BLKINFO)) { -// continue; -// } -// -// return false; -// } - } - - for(int32_t i = 0; i < numOfOutput; ++i) { - if (!validateExprColumnInfo(pTableInfo, pExpr[i], pTagCols)) { - return TSDB_CODE_QRY_INVALID_MSG; - } - } - - return true; -} - static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); @@ -8172,7 +8448,7 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t if (pColFilter->filterstr) { pColFilter->len = htobe64(pFilterMsg->len); - pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + pColFilter->pz = (int64_t)taosMemoryCalloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator if (pColFilter->pz == 0) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -8194,57 +8470,79 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, int32_t scale, int32_t precision, const char* name) { SResSchema s = {0}; s.scale = scale; - s.precision = precision; s.type = type; s.bytes = bytes; s.colId = slotId; + s.precision = precision; strncpy(s.name, name, tListLen(s.name)); return s; } -SExprInfo* createExprInfo(SNodeList* pNodeList, int32_t* numOfExprs) { +SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); + int32_t numOfGroupKeys = 0; + if (pGroupKeys != NULL) { + numOfGroupKeys = LIST_LENGTH(pGroupKeys); + } - *numOfExprs = numOfFuncs; - SExprInfo* pExprs = calloc(numOfFuncs, sizeof(SExprInfo)); + *numOfExprs = numOfFuncs + numOfGroupKeys; + SExprInfo* pExprs = taosMemoryCalloc(*numOfExprs, sizeof(SExprInfo)); - for(int32_t i = 0; i < numOfFuncs; ++i) { - SExprInfo* pExp = &pExprs[i]; + for(int32_t i = 0; i < (*numOfExprs); ++i) { + STargetNode* pTargetNode = NULL; + if (i < numOfFuncs) { + pTargetNode = (STargetNode*)nodesListGetNode(pNodeList, i); + } else { + pTargetNode = (STargetNode*)nodesListGetNode(pGroupKeys, i - numOfFuncs); + } + + SExprInfo* pExp = &pExprs[pTargetNode->slotId]; - pExp->pExpr = calloc(1, sizeof(tExprNode)); + pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); pExp->pExpr->_function.num = 1; + pExp->pExpr->_function.functionId = -1; - pExp->base.pParam = calloc(1, sizeof(SFunctParam)); + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); pExp->base.numOfParams = 1; - pExp->base.pParam[0].pCol = calloc(1, sizeof(SColumn)); + pExp->base.pParam[0].pCol = taosMemoryCalloc(1, sizeof(SColumn)); SColumn* pCol = pExp->base.pParam[0].pCol; - STargetNode* pTargetNode = (STargetNode*) nodesListGetNode(pNodeList, i); - ASSERT(pTargetNode->slotId == i); - - SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; - - SDataType *pType = &pFuncNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, - pType->scale, pType->precision, pFuncNode->node.aliasName); - - pExp->pExpr->_function.pFunctNode = pFuncNode; - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, tListLen(pExp->pExpr->_function.functionName)); - - // TODO: value parameter needs to be handled - int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); - for(int32_t j = 0; j < numOfParam; ++j) { - SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); - SColumnNode* pcn = (SColumnNode*)p1; - - pCol->slotId = pcn->slotId; - pCol->bytes = pcn->node.resType.bytes; - pCol->type = pcn->node.resType.type; - pCol->scale = pcn->node.resType.scale; - pCol->precision = pcn->node.resType.precision; - pCol->dataBlockId = pcn->dataBlockId; + // it is a project query, or group by column + if (nodeType(pTargetNode->pExpr) == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*) pTargetNode->pExpr; + + SDataType* pType = &pColNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pColNode->colName); + pCol->slotId = pColNode->slotId; + pCol->bytes = pType->bytes; + pCol->type = pType->type; + pCol->scale = pType->scale; + pCol->precision = pType->precision; + } else { + SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; + + SDataType* pType = &pFuncNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); + + pExp->pExpr->_function.functionId = pFuncNode->funcId; + pExp->pExpr->_function.pFunctNode = pFuncNode; + strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, tListLen(pExp->pExpr->_function.functionName)); + + // TODO: value parameter needs to be handled + int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); + for (int32_t j = 0; j < numOfParam; ++j) { + SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); + SColumnNode* pcn = (SColumnNode*)p1; + + pCol->slotId = pcn->slotId; + pCol->bytes = pcn->node.resType.bytes; + pCol->type = pcn->node.resType.type; + pCol->scale = pcn->node.resType.scale; + pCol->precision = pcn->node.resType.precision; + pCol->dataBlockId = pcn->dataBlockId; + } } } @@ -8252,40 +8550,34 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, int32_t* numOfExprs) { } static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { - SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo)); + SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; - char* p = calloc(1, 128); + char* p = taosMemoryCalloc(1, 128); snprintf(p, 128, "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, queryId); pTaskInfo->id.str = strdup(p); return pTaskInfo; } -static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId); +static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableGroupInfo *pTableGroupInfo, uint64_t queryId, uint64_t taskId); static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractScanColumnId(SNodeList* pNodeList); +static SArray* extractColumnInfo(SNodeList* pNodeList); SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { - if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_PROJECT) { // ignore the project node - pPhyNode = nodesListGetNode(pPhyNode->pChildren, 0); - } - if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pPhyNode)) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols); - tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, (uint64_t)queryId, taskId); - - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); - return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, - pScanPhyNode->reverse, pTaskInfo); + tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); + return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pExchange->node.pOutputDataBlockDesc); @@ -8293,18 +8585,28 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. - STableGroupInfo groupInfo = {0}; - - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); - SArray* tableIdList = extractTableIdList(&groupInfo); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createOutputBuf_rv1(pScanPhyNode->node.pOutputDataBlockDesc); SArray* colList = extractScanColumnId(pScanPhyNode->pScanCols); SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, colList, tableIdList, pTaskInfo); - taosArrayDestroy(tableIdList); return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == nodeType(pPhyNode)) { + SSystemTableScanPhysiNode * pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createOutputBuf_rv1(pSysScanPhyNode->scan.node.pOutputDataBlockDesc); + + struct SScanPhysiNode* pScanNode = &pSysScanPhyNode->scan; + SArray* colList = extractScanColumnId(pScanNode->pScanCols); + + SOperatorInfo* pOperator = createSysTableScanOperatorInfo(pHandle->meta, pResBlock, &pScanNode->tableName, + pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, + colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); + return pOperator; + } else { + ASSERT(0); } } @@ -8317,7 +8619,7 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(((SProjectPhysiNode*)pPhyNode)->pProjections, &num); + SExprInfo* pExprInfo = createExprInfo(((SProjectPhysiNode*)pPhyNode)->pProjections, NULL, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); return createProjectOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo); } @@ -8330,9 +8632,17 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(((SAggPhysiNode*)pPhyNode)->pAggFuncs, &num); + + SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; + SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); + + if (pAggNode->pGroupKeys != NULL) { + SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys); + return createGroupOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pTaskInfo, NULL); + } else { + return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); + } } } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == nodeType(pPhyNode)) { size_t size = LIST_LENGTH(pPhyNode->pChildren); @@ -8345,10 +8655,12 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->pFuncs, &num); + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = 'a', .slidingUnit = 'a'}; + SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, .offset = pIntervalPhyNode->offset}; return createIntervalOperatorInfo(op, pExprInfo, num, pResBlock, &interval, pTableGroupInfo, pTaskInfo); } } /*else if (pPhyNode->info.type == OP_MultiTableAggregate) { @@ -8368,7 +8680,7 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa cond.order = pTableScanNode->scan.order; cond.numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); if (cond.colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -8413,6 +8725,32 @@ SArray* extractScanColumnId(SNodeList* pNodeList) { return pList; } +SArray* extractColumnInfo(SNodeList* pNodeList) { + size_t numOfCols = LIST_LENGTH(pNodeList); + SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for(int32_t i = 0; i < numOfCols; ++i) { + STargetNode* pNode = (STargetNode*) nodesListGetNode(pNodeList, i); + SColumnNode* pColNode = (SColumnNode*) pNode->pExpr; + + SColumn c = {0}; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.precision = pColNode->node.resType.precision; + c.scale = pColNode->node.resType.scale; + + taosArrayPush(pList, &c); + } + + return pList; +} + int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { int32_t code = 0; if (tableType == TSDB_SUPER_TABLE) { @@ -8442,22 +8780,20 @@ SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo) { return tableIdList; } -tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { - STableGroupInfo groupInfo = {0}; - +tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableGroupInfo *pTableGroupInfo, uint64_t queryId, uint64_t taskId) { uint64_t uid = pTableScanNode->scan.uid; - int32_t code = doCreateTableGroup(pHandle->meta, pTableScanNode->scan.tableType, uid, &groupInfo, queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pTableScanNode->scan.tableType, uid, pTableGroupInfo, queryId, taskId); if (code != TSDB_CODE_SUCCESS) { goto _error; } - if (groupInfo.numOfTables == 0) { + if (pTableGroupInfo->numOfTables == 0) { code = 0; qDebug("no table qualified for query, TID:0x%"PRIx64", QID:0x%"PRIx64, taskId, queryId); goto _error; } - return createDataReaderImpl(pTableScanNode, &groupInfo, pHandle->reader, queryId, taskId); + return createDataReaderImpl(pTableScanNode, pTableGroupInfo, pHandle->reader, queryId, taskId); _error: terrno = code; @@ -8476,6 +8812,11 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead STableGroupInfo group = {0}; (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &group); + if (NULL == (*pTaskInfo)->pRoot) { + code = terrno; + goto _complete; + } + if ((*pTaskInfo)->pRoot == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; @@ -8484,7 +8825,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead return code; _complete: - tfree(*pTaskInfo); + taosMemoryFreeClear(*pTaskInfo); terrno = code; return code; @@ -8495,7 +8836,7 @@ int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int return TSDB_CODE_SUCCESS; } - *dst = calloc(filterNum, sizeof(*src)); + *dst = taosMemoryCalloc(filterNum, sizeof(*src)); if (*dst == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -8504,11 +8845,11 @@ int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int for (int32_t i = 0; i < filterNum; i++) { if ((*dst)[i].filterstr && dst[i]->len > 0) { - void *pz = calloc(1, (size_t)(*dst)[i].len + 1); + void *pz = taosMemoryCalloc(1, (size_t)(*dst)[i].len + 1); if (pz == NULL) { if (i == 0) { - free(*dst); + taosMemoryFree(*dst); } else { freeColumnFilterInfo(*dst, i); } @@ -8568,7 +8909,7 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { } //int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { -// *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); +// *pFilterInfo = taosMemoryCalloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); // if (*pFilterInfo == NULL) { // return TSDB_CODE_QRY_OUT_OF_MEMORY; // } @@ -8581,7 +8922,7 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { // pFilter->info = pCols[i]; // // pFilter->numOfFilters = pCols[i].flist.numOfFilters; -// pFilter->pFilters = calloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); +// pFilter->pFilters = taosMemoryCalloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); // if (pFilter->pFilters == NULL) { // return TSDB_CODE_QRY_OUT_OF_MEMORY; // } @@ -8623,11 +8964,11 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil // if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) { // taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q)); // } -// tfree(pFilterInfo[i].pFilters); +// taosMemoryFreeClear(pFilterInfo[i].pFilters); // } // } // -// tfree(pFilterInfo); +// taosMemoryFreeClear(pFilterInfo); return NULL; } @@ -8687,7 +9028,7 @@ static void doUpdateExprColumnIndex(STaskAttr *pQueryAttr) { } } -void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo) { +void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); // the minimum number of rows for projection query @@ -8708,7 +9049,7 @@ void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo) { } pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); - pResultInfo->total = 0; + pResultInfo->totalRows = 0; } //TODO refactor @@ -8719,11 +9060,11 @@ void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { for (int32_t i = 0; i < numOfFilters; i++) { if (pFilter[i].filterstr && pFilter[i].pz) { - free((void*)(pFilter[i].pz)); + taosMemoryFree((void*)(pFilter[i].pz)); } } - free(pFilter); + taosMemoryFree(pFilter); } static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) { @@ -8757,9 +9098,9 @@ void doDestroyTask(SExecTaskInfo *pTaskInfo) { // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); - tfree(pTaskInfo->sql); - tfree(pTaskInfo->id.str); - tfree(pTaskInfo); + taosMemoryFreeClear(pTaskInfo->sql); + taosMemoryFreeClear(pTaskInfo->id.str); + taosMemoryFreeClear(pTaskInfo); } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { @@ -8812,30 +9153,6 @@ int32_t checkForQueryBuf(size_t numOfTables) { return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; } -bool checkNeedToCompressQueryCol(SQInfo *pQInfo) { - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SSDataBlock* pRes = pRuntimeEnv->outputBuf; - - if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) <= 0) { - return false; - } - - int32_t numOfRows = pQueryAttr->pExpr2 ? GET_NUM_OF_RESULTS(pRuntimeEnv) : pRes->info.rows; - int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - int32_t colSize = pColRes->info.bytes * numOfRows; - if (NEEDTO_COMPRESS_QUERY(colSize)) { - return true; - } - } - - return false; -} - void releaseQueryBuf(size_t numOfTables) { if (tsQueryBufferSizeBytes < 0) { return; diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 28319469ccc0ad039490de31cea5d55a6350198e..1b55467c8587e2b2132bae8b3f549bb7a7c39035 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -209,7 +209,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { newLen += 4; } - char* p = realloc(pHashObj->pBucket, POINTER_BYTES * newLen); + char* p = taosMemoryRealloc(pHashObj->pBucket, POINTER_BYTES * newLen); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -219,7 +219,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { pHashObj->numOfAlloc = newLen; } - SLHashBucket* pBucket = calloc(1, sizeof(SLHashBucket)); + SLHashBucket* pBucket = taosMemoryCalloc(1, sizeof(SLHashBucket)); pHashObj->pBucket[pHashObj->numOfBuckets] = pBucket; pBucket->pPageIdList = taosArrayInit(2, sizeof(int32_t)); @@ -241,7 +241,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { } SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_t numOfTuplePerPage) { - SLHashObj* pHashObj = calloc(1, sizeof(SLHashObj)); + SLHashObj* pHashObj = taosMemoryCalloc(1, sizeof(SLHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -265,12 +265,12 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ pHashObj->tuplesPerPage = numOfTuplePerPage; pHashObj->numOfAlloc = 4; // initial allocated array list - pHashObj->pBucket = calloc(pHashObj->numOfAlloc, POINTER_BYTES); + pHashObj->pBucket = taosMemoryCalloc(pHashObj->numOfAlloc, POINTER_BYTES); code = doAddNewBucket(pHashObj); if (code != TSDB_CODE_SUCCESS) { destroyDiskbasedBuf(pHashObj->pBuf); - tfree(pHashObj); + taosMemoryFreeClear(pHashObj); terrno = code; return NULL; } @@ -282,11 +282,11 @@ void* tHashCleanup(SLHashObj* pHashObj) { destroyDiskbasedBuf(pHashObj->pBuf); for(int32_t i = 0; i < pHashObj->numOfBuckets; ++i) { taosArrayDestroy(pHashObj->pBucket[i]->pPageIdList); - tfree(pHashObj->pBucket[i]); + taosMemoryFreeClear(pHashObj->pBucket[i]); } - tfree(pHashObj->pBucket); - tfree(pHashObj); + taosMemoryFreeClear(pHashObj->pBucket); + taosMemoryFreeClear(pHashObj); return NULL; } diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 4012c3926ef78597dc4b3f5d190549658cbe7b63..981da0415e63684841099f24729891f0e8ccb5a1 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -29,7 +29,7 @@ #define FREE_HASH_NODE(_n) \ do { \ - tfree(_n); \ + taosMemoryFreeClear(_n); \ } while (0); typedef struct SHNode { @@ -40,7 +40,7 @@ typedef struct SHNode { typedef struct SSHashObj { SHNode **hashList; size_t capacity; // number of slots - size_t size; // number of elements in hash table + int64_t size; // number of elements in hash table _hash_fn_t hashFp; // hash function _equal_fn_t equalFp; // equal function int32_t keyLen; @@ -62,7 +62,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t capacity = 4; } - SSHashObj* pHashObj = (SSHashObj*) calloc(1, sizeof(SSHashObj)); + SSHashObj* pHashObj = (SSHashObj*) taosMemoryCalloc(1, sizeof(SSHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -78,9 +78,9 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t pHashObj->keyLen = keyLen; pHashObj->dataLen = dataLen; - pHashObj->hashList = (SHNode **)calloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHNode **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { - free(pHashObj); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -91,11 +91,11 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64(&pHashObj->size); + return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); } static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHNode *pNewNode = malloc(sizeof(SHNode) + keyLen + dsize); + SHNode *pNewNode = taosMemoryMalloc(sizeof(SHNode) + keyLen + dsize); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -120,7 +120,7 @@ void taosHashTableResize(SSHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { // qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; @@ -287,7 +287,7 @@ void tSimpleHashCleanup(SSHashObj *pHashObj) { } tSimpleHashClear(pHashObj); - tfree(pHashObj->hashList); + taosMemoryFreeClear(pHashObj->hashList); } size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 08bab762be048b5a6db1a1205a00853e667e1eb8..85ba462c9a87de3af56089dd09fc384f7ab3af19 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -65,7 +65,7 @@ typedef struct SSortHandle { static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.numOfCols = numOfCols; @@ -91,7 +91,7 @@ static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { * @return */ SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr) { - SSortHandle* pSortHandle = calloc(1, sizeof(SSortHandle)); + SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle)); pSortHandle->type = type; pSortHandle->pageSize = pageSize; @@ -118,8 +118,8 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { } destroyDiskbasedBuf(pSortHandle->pBuf); - tfree(pSortHandle->idStr); - tfree(pSortHandle); + taosMemoryFreeClear(pSortHandle->idStr); + taosMemoryFreeClear(pSortHandle); } int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { @@ -127,7 +127,7 @@ int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { } static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId) { - SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource)); + SExternalMemSource* pSource = taosMemoryCalloc(1, sizeof(SExternalMemSource)); if (pSource == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -182,7 +182,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { start = stop + 1; } - blockDataClearup(pDataBlock); + blockDataCleanup(pDataBlock); SSDataBlock* pBlock = createOneDataBlock(pDataBlock); int32_t code = doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId); @@ -233,7 +233,7 @@ static int32_t sortComparClearup(SMsortComparParam* cmpParam) { for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { SExternalMemSource* pSource = cmpParam->pSources[i]; blockDataDestroy(pSource->src.pBlock); - tfree(pSource); + taosMemoryFreeClear(pSource); } cmpParam->numOfSources = 0; @@ -312,7 +312,7 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa } static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity) { - blockDataClearup(pHandle->pDataBlock); + blockDataCleanup(pHandle->pDataBlock); while(1) { if (cmpParam->numOfSources == pHandle->numOfCompletedSources) { @@ -478,7 +478,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); - blockDataClearup(pDataBlock); + blockDataCleanup(pDataBlock); } tMergeTreeDestroy(pHandle->pMergeTree); @@ -570,7 +570,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { } } - tfree(source); + taosMemoryFreeClear(source); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 6737f57bbf84c43a7115c37436bbc22355b26a85..7fb9b2ad7e9bbaaffd8f7ddcd37f90a40114ad2d 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -62,7 +62,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -70,8 +70,8 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo.info.type = TSDB_DATA_TYPE_INT; colInfo.info.bytes = sizeof(int32_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); @@ -82,11 +82,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { // // colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t); // colInfo1.varmeta.length = 0; -// colInfo1.varmeta.offset = static_cast(calloc(1, numOfRows * sizeof(int32_t))); +// colInfo1.varmeta.offset = static_cast(taosMemoryCalloc(1, numOfRows * sizeof(int32_t))); // // taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); } else { - blockDataClearup(pInfo->pBlock); + blockDataCleanup(pInfo->pBlock); } SSDataBlock* pBlock = pInfo->pBlock; @@ -128,7 +128,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -136,8 +136,8 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; colInfo.info.bytes = sizeof(int64_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); -// colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); +// colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); @@ -146,12 +146,12 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo1.info.bytes = 4; colInfo1.info.colId = 2; - colInfo1.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo1.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo1.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo1.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); } else { - blockDataClearup(pInfo->pBlock); + blockDataCleanup(pInfo->pBlock); } SSDataBlock* pBlock = pInfo->pBlock; @@ -195,7 +195,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_t rowsPerPage, int32_t type, int32_t numOfCols) { - SOperatorInfo* pOperator = static_cast(calloc(1, sizeof(SOperatorInfo))); + SOperatorInfo* pOperator = static_cast(taosMemoryCalloc(1, sizeof(SOperatorInfo))); pOperator->name = "dummyInputOpertor4Test"; if (numOfCols == 1) { @@ -204,7 +204,7 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_ pOperator->getNextFn = get2ColsDummyBlock; } - SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); + SDummyInputInfo *pInfo = (SDummyInputInfo*) taosMemoryCalloc(1, sizeof(SDummyInputInfo)); pInfo->totalPages = numOfBlocks; pInfo->startVal = startVal; pInfo->numOfRowsPerPage = rowsPerPage; @@ -958,11 +958,11 @@ TEST(testCase, inMem_sort_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); taosArrayPush(pExprInfo, &exp1); @@ -1002,10 +1002,10 @@ int32_t cmp(const void* p1, const void* p2) { #if 0 TEST(testCase, external_sort_Test) { #if 0 - su* v = static_cast(calloc(1000000, sizeof(su))); + su* v = static_cast(taosMemoryCalloc(1000000, sizeof(su))); for(int32_t i = 0; i < 1000000; ++i) { v[i].v = taosRand(); - v[i].c = static_cast(malloc(4)); + v[i].c = static_cast(taosMemoryMalloc(4)); *(int32_t*) v[i].c = i; } @@ -1027,11 +1027,11 @@ TEST(testCase, external_sort_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); @@ -1071,8 +1071,8 @@ TEST(testCase, external_sort_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } @@ -1088,21 +1088,21 @@ TEST(testCase, sorted_merge_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 1, "count_result"); - exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp->base.pColumns->flag = TSDB_COL_NORMAL; exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; exp->base.numOfCols = 1; taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); int32_t numOfSources = 10; - SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*)); + SOperatorInfo** plist = (SOperatorInfo**) taosMemoryCalloc(numOfSources, sizeof(void*)); for(int32_t i = 0; i < numOfSources; ++i) { plist[i] = createDummyOperator(1, 1, 1, data_asc, 1); } @@ -1143,8 +1143,8 @@ TEST(testCase, sorted_merge_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } @@ -1160,18 +1160,18 @@ TEST(testCase, time_interval_Operator_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, "ts"); - exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp->base.pColumns->flag = TSDB_COL_NORMAL; exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8}; exp->base.numOfCols = 1; taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, 8, 2, "res1"); - exp1->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp1->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp1->base.pColumns->flag = TSDB_COL_NORMAL; exp1->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; exp1->base.numOfCols = 1; @@ -1221,11 +1221,11 @@ TEST(testCase, time_interval_Operator_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } #endif -#pragma GCC diagnostic pop +#pragma GCC diagnosti \ No newline at end of file diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 19567943954b07874eae5f5807b1d7066a1cfb30..586aed7a67c7705993f6d16bd68cad6662fa1d99 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -47,15 +47,15 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return NULL; } - SSDataBlock* pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); SColumnInfoData colInfo = {0}; colInfo.info.type = TSDB_DATA_TYPE_INT; colInfo.info.bytes = sizeof(int32_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->pageRows, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (pInfo->pageRows + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); taosArrayPush(pBlock->pDataBlock, &colInfo); @@ -203,12 +203,12 @@ TEST(testCase, external_mem_sort_Test) { SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); - _info* pInfo = (_info*) calloc(1, sizeof(_info)); + _info* pInfo = (_info*) taosMemoryCalloc(1, sizeof(_info)); pInfo->startVal = 100000; pInfo->pageRows = 1000; pInfo->count = 50; - SGenericSource* ps = static_cast(calloc(1, sizeof(SGenericSource))); + SGenericSource* ps = static_cast(taosMemoryCalloc(1, sizeof(SGenericSource))); ps->param = pInfo; tsortAddSource(phandle, ps); @@ -249,8 +249,8 @@ TEST(testCase, ordered_merge_sort_Test) { tsortSetComparFp(phandle, docomp); for(int32_t i = 0; i < 10; ++i) { - SGenericSource* p = static_cast(calloc(1, sizeof(SGenericSource))); - _info* c = static_cast<_info*>(calloc(1, sizeof(_info))); + SGenericSource* p = static_cast(taosMemoryCalloc(1, sizeof(SGenericSource))); + _info* c = static_cast<_info*>(taosMemoryCalloc(1, sizeof(_info))); c->count = 1; c->pageRows = 1000; c->startVal = 0; diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index a10a542b6b2e094166202c6cb299801576d15234..e1c7626f014b07b49f20d7b6150048793798da05 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -1,12 +1,57 @@ aux_source_directory(src FUNCTION_SRC) +list(REMOVE_ITEM FUNCTION_SRC src/udfd.c) add_library(function STATIC ${FUNCTION_SRC}) target_include_directories( function - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( function - PRIVATE os util common nodes -) \ No newline at end of file + PRIVATE os util common nodes scalar + PUBLIC uv_a +) + +add_executable(runUdf test/runUdf.c) +target_include_directories( + runUdf + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" + "${CMAKE_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + runUdf + PUBLIC uv_a + PRIVATE os util common nodes function +) + +add_library(udf1 MODULE test/udf1.c) +target_include_directories( + udf1 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +#SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) +add_executable(udfd src/udfd.c) +target_include_directories( + udfd + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + udfd + PUBLIC uv_a + PRIVATE os util common nodes function +) + diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 7ba7d7bdcccffd14e6f224e62ddf08e2dd8475bd..214204723fd5ec8c9ee3b09bf7c565e41c7627b1 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -23,7 +23,7 @@ extern "C" { #include "function.h" bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); -void functionFinalizer(SqlFunctionCtx *pCtx); +void functionFinalize(SqlFunctionCtx *pCtx); bool getCountFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); void countFunction(SqlFunctionCtx *pCtx); @@ -37,6 +37,16 @@ bool getMinmaxFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); void minFunction(SqlFunctionCtx* pCtx); void maxFunction(SqlFunctionCtx *pCtx); +bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); +void stddevFunction(SqlFunctionCtx* pCtx); +void stddevFinalize(SqlFunctionCtx* pCtx); + +bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); +void firstFunction(SqlFunctionCtx *pCtx); +void lastFunction(SqlFunctionCtx *pCtx); + +void valFunction(SqlFunctionCtx *pCtx); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/inc/tfill.h b/source/libs/function/inc/tfill.h index 81348fba1dd6529b7be2b8c3ff88dbf57323a6a8..b90dbf7799a904609a293861d9de68775b2ca100 100644 --- a/source/libs/function/inc/tfill.h +++ b/source/libs/function/inc/tfill.h @@ -61,7 +61,7 @@ typedef struct SFillInfo { SFillColInfo* pFillCol; // column info for fill operations SFillTagColInfo* pTags; // tags value for filling gap - void* handle; // for debug purpose + const char* id; } SFillInfo; int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); diff --git a/source/libs/function/inc/tscript.h b/source/libs/function/inc/tscript.h index 281fe6f67988a10df36b0124d4a613587de4ec57..823e2d61f2baa9a107c59608d757de0f673cc38b 100644 --- a/source/libs/function/inc/tscript.h +++ b/source/libs/function/inc/tscript.h @@ -70,7 +70,7 @@ typedef struct { SList *scriptEnvs; // int32_t mSize; // pool limit int32_t cSize; // current available size - pthread_mutex_t mutex; + TdThreadMutex mutex; } ScriptEnvPool; ScriptCtx* createScriptCtx(char *str, int8_t resType, int16_t resBytes); diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 163fbdf4bb204be1c00022505ea13cf32854ff99..72875239d2c7f335d08ccf1c3af73108e4b04ef9 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -20,68 +20,116 @@ extern "C" { #endif -#include "os.h" -#include "taoserror.h" +//====================================================================================== +//begin API to taosd and qworker +/** + * start udf dameon service + * @return error code + */ +int32_t startUdfService(); + +/** + * stop udf dameon service + * @return error code + */ +int32_t stopUdfService(); enum { - TSDB_UDF_FUNC_NORMAL = 0, - TSDB_UDF_FUNC_INIT, - TSDB_UDF_FUNC_FINALIZE, - TSDB_UDF_FUNC_MERGE, - TSDB_UDF_FUNC_DESTROY, - TSDB_UDF_FUNC_MAX_NUM + TSDB_UDF_TYPE_SCALAR = 0, + TSDB_UDF_TYPE_AGGREGATE = 1 }; -typedef struct SUdfInit { - int32_t maybe_null; /* 1 if function can return NULL */ - uint32_t decimals; /* for real functions */ - uint64_t length; /* For string functions */ - char* ptr; /* free pointer for function data */ - int32_t const_item; /* 0 if result is independent of arguments */ - - // script like lua/javascript - void* script_ctx; - void (*destroyCtxFunc)(void* script_ctx); -} SUdfInit; +enum { + TSDB_UDF_SCRIPT_BIN_LIB = 0, + TSDB_UDF_SCRIPT_LUA = 1, +}; typedef struct SUdfInfo { - int32_t functionId; // system assigned function id - int32_t funcType; // scalar function or aggregate function - int8_t resType; // result type - int16_t resBytes; // result byte - int32_t contLen; // content length - int32_t bufSize; // interbuf size - char* name; // function name - void* handle; // handle loaded in mem - void* funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr - - // for script like lua/javascript only - int isScript; - void* pScriptCtx; - - SUdfInit init; - char* content; - char* path; + char *udfName; // function name + int32_t udfType; // scalar function or aggregate function + int8_t scriptType; + char *path; + + int8_t resType; // result type + int16_t resBytes; // result byte + int32_t bufSize; //interbuf size + } SUdfInfo; +typedef void *UdfHandle; + +/** + * setup udf + * @param udf, in + * @param handle, out + * @return error code + */ +int32_t setupUdf(SUdfInfo* udf, UdfHandle *handle); + + +enum { + TSDB_UDF_STEP_NORMAL = 0, + TSDB_UDF_STEP_MERGE, + TSDb_UDF_STEP_FINALIZE, + TSDB_UDF_STEP_MAX_NUM +}; +/** + * call udf + * @param handle udf handle + * @param step + * @param state + * @param stateSize + * @param input + * @param newstate + * @param newStateSize + * @param output + * @return error code + */ + +//TODO: must change the following after metadata flow and data flow between qworker and udfd is well defined +typedef struct SUdfDataBlock { + char* data; + int32_t size; +} SUdfDataBlock; + +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newstate, + int32_t *newStateSize, SUdfDataBlock *output); + +/** + * tearn down udf + * @param handle + * @return + */ +int32_t teardownUdf(UdfHandle handle); + +// end API to taosd and qworker +//============================================================================================================================= +// TODO: Must change +// begin API to UDF writer. + // script -typedef int32_t (*scriptInitFunc)(void* pCtx); -typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, - int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, - int16_t oType, int16_t oBytes); -typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptDestroyFunc)(void* pCtx); +//typedef int32_t (*scriptInitFunc)(void* pCtx); +//typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, +// int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, +// int16_t oType, int16_t oBytes); +//typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); +//typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); +//typedef void (*scriptDestroyFunc)(void* pCtx); // dynamic lib -typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, - char* dataOutput, char* interBuf, char* tsOutput, int32_t* numOfOutput, int16_t oType, - int16_t oBytes, SUdfInit* buf); -typedef int32_t (*udfInitFunc)(SUdfInit* data); -typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfDestroyFunc)(SUdfInit* buf); +typedef int32_t (*TUdfInitFunc)(); +typedef void (*TUdfDestroyFunc)(); + +typedef void (*TUdfFunc)(int8_t step, + char *state, int32_t stateSize, SUdfDataBlock input, + char **newstate, int32_t *newStateSize, SUdfDataBlock *output); + +//typedef void (*udfMergeFunc)(char *data, int32_t numOfRows, char *dataOutput, int32_t* numOfOutput); +//typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output); + +// end API to UDF writer +//======================================================================================================================= #ifdef __cplusplus } diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h new file mode 100644 index 0000000000000000000000000000000000000000..5f757c1ef0a36d47d52a1b74670d3f1b888b676f --- /dev/null +++ b/source/libs/function/inc/tudfInt.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TUDF_INT_H +#define TDENGINE_TUDF_INT_H + +#ifdef __cplusplus +extern "C" { +#endif + +//TODO replaces them with fnDebug +//#define debugPrint(...) taosPrintLog("Function", DEBUG_INFO, 135, __VA_ARGS__) +#define debugPrint(...) {fprintf(stderr, __VA_ARGS__);fprintf(stderr, "\n");} +enum { + UDF_TASK_SETUP = 0, + UDF_TASK_CALL = 1, + UDF_TASK_TEARDOWN = 2 + +}; + +typedef struct SUdfSetupRequest { + char udfName[16]; // + int8_t scriptType; // 0:c, 1: lua, 2:js + int8_t udfType; //udaf, udf + int16_t pathSize; + char *path; +} SUdfSetupRequest; + +typedef struct SUdfSetupResponse { + int64_t udfHandle; +} SUdfSetupResponse; + + +typedef struct SUdfCallRequest { + int64_t udfHandle; + int8_t step; + + int32_t inputBytes; + char *input; + + int32_t stateBytes; + char *state; +} SUdfCallRequest; + + +typedef struct SUdfCallResponse { + int32_t outputBytes; + char *output; + int32_t newStateBytes; + char *newState; +} SUdfCallResponse; + + +typedef struct SUdfTeardownRequest { + int64_t udfHandle; +} SUdfTeardownRequest; + + +typedef struct SUdfTeardownResponse { +} SUdfTeardownResponse; + +typedef struct SUdfRequest { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + void *subReq; +} SUdfRequest; + +typedef struct SUdfResponse { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + int32_t code; + void *subRsp; +} SUdfResponse; + +int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest); +int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response); +int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request); +int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TUDF_INT_H diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h new file mode 100644 index 0000000000000000000000000000000000000000..4d2aeb7049c21a07d1875c2ad9169ee3b5c9212b --- /dev/null +++ b/source/libs/function/inc/udfc.h @@ -0,0 +1,110 @@ +// +// Created by shenglian on 28/02/22. +// + +#ifndef UDF_UDF_H +#define UDF_UDF_H + +#include +#define DEBUG +#ifdef DEBUG +#define debugPrint(...) fprintf(__VA_ARGS__) +#else +#define debugPrint(...) /**/ +#endif + +enum { + UDF_TASK_SETUP = 0, + UDF_TASK_CALL = 1, + UDF_TASK_TEARDOWN = 2 + +}; + +typedef struct SSDataBlock{ + char *data; + int32_t size; +} SSDataBlock; + +typedef struct SUdfInfo { + char *udfName; + char *path; +} SUdfInfo; + +typedef void *UdfHandle; + +int32_t startUdfService(); + +int32_t stopUdfService(); + +//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles); + +int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle); + +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, + int32_t *newStateSize, SSDataBlock *output); + +int32_t teardownUdf(UdfHandle handle); + +typedef struct SUdfSetupRequest { + char udfName[16]; // + int8_t scriptType; // 0:c, 1: lua, 2:js + int8_t udfType; //udaf, udf, udtf + int16_t pathSize; + char *path; +} SUdfSetupRequest; + +typedef struct SUdfSetupResponse { + int64_t udfHandle; +} SUdfSetupResponse; + + +typedef struct SUdfCallRequest { + int64_t udfHandle; + int8_t step; + + int32_t inputBytes; + char *input; + + int32_t stateBytes; + char *state; +} SUdfCallRequest; + + +typedef struct SUdfCallResponse { + int32_t outputBytes; + char *output; + int32_t newStateBytes; + char *newState; +} SUdfCallResponse; + + +typedef struct SUdfTeardownRequest { + int64_t udfHandle; +} SUdfTeardownRequest; + + +typedef struct SUdfTeardownResponse { +} SUdfTeardownResponse; + +typedef struct SUdfRequest { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + void *subReq; +} SUdfRequest; + +typedef struct SUdfResponse { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + int32_t code; + void *subRsp; +} SUdfResponse; + +int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest); +int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response); +int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request); +int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse); +#endif //UDF_UDF_H diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index edb0acf07508c520abddf51e06e1e75ea30264d6..70b2a48da7944e7e80a65ce3128fc9e992c2fbf1 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -15,6 +15,7 @@ #include "builtins.h" #include "builtinsimpl.h" +#include "scalar.h" #include "taoserror.h" #include "tdatablock.h" @@ -29,7 +30,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getCountFuncEnv, .initFunc = functionSetup, .processFunc = countFunction, - .finalizeFunc = functionFinalizer + .finalizeFunc = functionFinalize }, { .name = "sum", @@ -39,7 +40,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getSumFuncEnv, .initFunc = functionSetup, .processFunc = sumFunction, - .finalizeFunc = functionFinalizer + .finalizeFunc = functionFinalize }, { .name = "min", @@ -49,7 +50,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getMinmaxFuncEnv, .initFunc = minFunctionSetup, .processFunc = minFunction, - .finalizeFunc = functionFinalizer + .finalizeFunc = functionFinalize }, { .name = "max", @@ -59,7 +60,227 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getMinmaxFuncEnv, .initFunc = maxFunctionSetup, .processFunc = maxFunction, - .finalizeFunc = functionFinalizer + .finalizeFunc = functionFinalize + }, + { + .name = "stddev", + .type = FUNCTION_TYPE_STDDEV, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getStddevFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "percentile", + .type = FUNCTION_TYPE_PERCENTILE, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "apercentile", + .type = FUNCTION_TYPE_APERCENTILE, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "top", + .type = FUNCTION_TYPE_TOP, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "bottom", + .type = FUNCTION_TYPE_BOTTOM, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "spread", + .type = FUNCTION_TYPE_SPREAD, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "last_row", + .type = FUNCTION_TYPE_LAST_ROW, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "first", + .type = FUNCTION_TYPE_FIRST, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = firstFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "last", + .type = FUNCTION_TYPE_LAST, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "abs", + .type = FUNCTION_TYPE_ABS, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = absFunction, + .finalizeFunc = NULL + }, + { + .name = "log", + .type = FUNCTION_TYPE_LOG, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = logFunction, + .finalizeFunc = NULL + }, + { + .name = "power", + .type = FUNCTION_TYPE_POW, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = powFunction, + .finalizeFunc = NULL + }, + { + .name = "sqrt", + .type = FUNCTION_TYPE_SQRT, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = sqrtFunction, + .finalizeFunc = NULL + }, + { + .name = "ceil", + .type = FUNCTION_TYPE_CEIL, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = ceilFunction, + .finalizeFunc = NULL + }, + { + .name = "floor", + .type = FUNCTION_TYPE_FLOOR, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = floorFunction, + .finalizeFunc = NULL + }, + { + .name = "round", + .type = FUNCTION_TYPE_ROUND, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = roundFunction, + .finalizeFunc = NULL + }, + { + .name = "sin", + .type = FUNCTION_TYPE_SIN, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = sinFunction, + .finalizeFunc = NULL + }, + { + .name = "cos", + .type = FUNCTION_TYPE_COS, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = cosFunction, + .finalizeFunc = NULL + }, + { + .name = "tan", + .type = FUNCTION_TYPE_TAN, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = tanFunction, + .finalizeFunc = NULL + }, + { + .name = "asin", + .type = FUNCTION_TYPE_ASIN, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = asinFunction, + .finalizeFunc = NULL + }, + { + .name = "acos", + .type = FUNCTION_TYPE_ACOS, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = acosFunction, + .finalizeFunc = NULL + }, + { + .name = "atan", + .type = FUNCTION_TYPE_ATAN, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = atanFunction, + .finalizeFunc = NULL }, { .name = "concat", @@ -98,6 +319,8 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType }; break; } + case FUNCTION_TYPE_FIRST: + case FUNCTION_TYPE_LAST: case FUNCTION_TYPE_MIN: case FUNCTION_TYPE_MAX: { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f0f00434f09cc937823a8d8797236e19b76d8368..610e5a0bb231a3f1693f4d18869fa53972ecfcdd 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -50,7 +50,7 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { static void doFinalizer(SResultRowEntryInfo* pResInfo) { cleanupResultRowEntry(pResInfo); } -void functionFinalizer(SqlFunctionCtx *pCtx) { +void functionFinalize(SqlFunctionCtx *pCtx) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); doFinalizer(pResInfo); } @@ -68,13 +68,12 @@ void countFunction(SqlFunctionCtx *pCtx) { int32_t numOfElem = 0; /* - * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->isAggSet == true; - * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->isAggSet == true; - * 3. for primary key column, pCtx->hasNull always be false, pCtx->isAggSet == false; + * 1. column data missing (schema modified) causes pInputCol->hasNull == true. pInput->colDataAggIsSet == true; + * 2. for general non-primary key columns, pInputCol->hasNull may be true or false, pInput->colDataAggIsSet == true; + * 3. for primary key column, pInputCol->hasNull always be false, pInput->colDataAggIsSet == false; */ SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - if (pInput->colDataAggIsSet && pInput->totalRows == pInput->numOfRows) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; ASSERT(numOfElem >= 0); @@ -169,7 +168,7 @@ void sumFunction(SqlFunctionCtx *pCtx) { SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); } -bool getSumFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { +bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SSumRes); return true; } @@ -261,8 +260,7 @@ bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { return true; } -bool getMinmaxFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - SNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); +bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(int64_t); return true; } @@ -274,34 +272,34 @@ bool getMinmaxFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - __ctx->fpSet.process(__ctx); \ + __ctx->fpSet.process(__ctx); \ } \ } while (0); -#define DO_UPDATE_SUBSID_RES(ctx, ts) \ - do { \ +#define DO_UPDATE_SUBSID_RES(ctx, ts) \ + do { \ for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \ - if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - __ctx->fpSet.process(__ctx); \ - } \ + SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \ + if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ + __ctx->tag.i = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ + } \ + __ctx->fpSet.process(__ctx); \ + } \ } while (0) #define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_SUBSID_RES(ctx, _ts); \ - (num) += 1; \ - } \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ + DO_UPDATE_SUBSID_RES(ctx, _ts); \ + (num) += 1; \ + } \ } while (0) -#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ +#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ do { \ - _t* d = (_t*)((_col)->pData); \ + _t *d = (_t *)((_col)->pData); \ for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \ if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ continue; \ @@ -441,4 +439,193 @@ void minFunction(SqlFunctionCtx *pCtx) { void maxFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = doMinMaxHelper(pCtx, 0); SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); -} \ No newline at end of file +} + +typedef struct STopBotRes { + int32_t num; +} STopBotRes; + +bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0); + int32_t bytes = pColNode->node.resType.bytes; + SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1); + return true; +} + +typedef struct SStddevRes { + int64_t count; + union {double quadraticDSum; int64_t quadraticISum;}; + union {double dsum; int64_t isum;}; +} SStddevRes; + +bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SStddevRes); + return true; +} + +void stddevFunction(SqlFunctionCtx* pCtx) { + int32_t numOfElem = 0; + + // Only the pre-computing information loaded and actual data does not loaded + SInputColumnInfoData* pInput = &pCtx->input; + SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; + int32_t type = pInput->pData[0]->info.type; + + SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + +// } else { // computing based on the true data block + SColumnInfoData* pCol = pInput->pData[0]; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + switch(type) { + case TSDB_DATA_TYPE_INT: { + int32_t* plist = (int32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + } + break; + } + + // data in the check operation are all null, not output + SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); +} + +void stddevFinalize(SqlFunctionCtx* pCtx) { + functionFinalize(pCtx); + + SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + double res = pStddevRes->quadraticISum/pStddevRes->count - (pStddevRes->isum / pStddevRes->count) * (pStddevRes->isum / pStddevRes->count); +} + + + + +bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); + pEnv->calcMemSize = pNode->node.resType.bytes; + return true; +} + +// TODO fix this +// This ordinary first function only handle the data block in ascending order +void firstFunction(SqlFunctionCtx *pCtx) { + if (pCtx->order == TSDB_ORDER_DESC) { + return; + } + + int32_t numOfElems = 0; + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + // All null data column, return directly. + if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { + ASSERT(pInputCol->hasNull == true); + return; + } + + // Check for the first not null data + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, NULL)) { + continue; + } + + char* data = colDataGetData(pInputCol, i); + memcpy(buf, data, pInputCol->info.bytes); + // TODO handle the subsidary value +// if (pCtx->ptsList != NULL) { +// TSKEY k = GET_TS_DATA(pCtx, i); +// DO_UPDATE_TAG_COLUMNS(pCtx, k); +// } + + pResInfo->hasResult = DATA_SET_FLAG; + pResInfo->complete = true; + + numOfElems++; + break; + } + + SET_VAL(pResInfo, numOfElems, 1); +} + +void lastFunction(SqlFunctionCtx *pCtx) { + if (pCtx->order != TSDB_ORDER_DESC) { + return; + } + + int32_t numOfElems = 0; + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + + SColumnInfoData* pInputCol = pInput->pData[0]; + + // All null data column, return directly. + if (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) { + ASSERT(pInputCol->hasNull == true); + return; + } + + if (pCtx->order == TSDB_ORDER_DESC) { + for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, NULL)) { + continue; + } + + char* data = colDataGetData(pInputCol, i); + memcpy(buf, data, pInputCol->info.bytes); + +// TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; +// DO_UPDATE_TAG_COLUMNS(pCtx, ts); + + pResInfo->hasResult = DATA_SET_FLAG; + pResInfo->complete = true; // set query completed on this column + numOfElems++; + break; + } + } else { // ascending order + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, NULL)) { + continue; + } + + char* data = colDataGetData(pInputCol, i); + TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; + + if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { + pResInfo->hasResult = DATA_SET_FLAG; + memcpy(buf, data, pCtx->inputBytes); + + *(TSKEY*)buf = ts; +// DO_UPDATE_TAG_COLUMNS(pCtx, ts); + } + + numOfElems++; + break; + } + } + + SET_VAL(pResInfo, numOfElems, 1); +} + +void valFunction(SqlFunctionCtx *pCtx) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + + SColumnInfoData* pInputCol = pCtx->input.pData[0]; + memcpy(buf, pInputCol->pData, pInputCol->info.bytes); +} diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 41b0126a073ef0e22c253406af592d9f1f46621d..10ac8bbf43cc59fbc23ad3bdf29ad1051cf16b5d 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -26,7 +26,7 @@ typedef struct SFuncMgtService { } SFuncMgtService; static SFuncMgtService gFunMgtService; -static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT; +static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; static int32_t initFunctionCode = 0; static void doInitFunctionHashTable() { @@ -45,7 +45,7 @@ static void doInitFunctionHashTable() { } int32_t fmFuncMgtInit() { - pthread_once(&functionHashTableInit, doInitFunctionHashTable); + taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable); return initFunctionCode; } @@ -97,7 +97,7 @@ bool fmIsAggFunc(int32_t funcId) { void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; - if (m != NULL && atomic_val_compare_exchange_ptr(&gFunMgtService.pFuncNameHashTable, m, 0) == m) { + if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { taosHashCleanup(m); } } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 4360515328d286304688a9949040f3e2df86335b..05ed30c61a231176739eb962ad53d1c5b17aa94d 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -1929,7 +1929,7 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { // set the corresponding tag data for each record // todo check malloc failure -// char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); +// char **pData = taosMemoryCalloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); // for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { // pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; // } @@ -1943,7 +1943,7 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { // } // } -// tfree(pData); +// taosMemoryFreeClear(pData); } /* @@ -2422,7 +2422,7 @@ static void apercentile_finalizer(SqlFunctionCtx *pCtx) { // double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); // // memcpy(pCtx->pOutput, res, sizeof(double)); -// free(res); +// taosMemoryFree(res); // } else { // setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); // return; @@ -2433,7 +2433,7 @@ static void apercentile_finalizer(SqlFunctionCtx *pCtx) { double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); memcpy(pCtx->pOutput, res, sizeof(double)); - free(res); + taosMemoryFree(res); } else { // no need to free setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; @@ -4004,7 +4004,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi char* outputBuf = NULL; if (comp) { - outputBuf = malloc(originalLen); + outputBuf = taosMemoryMalloc(originalLen); size_t actualLen = compLen; const char* compStr = tbufReadBinary(&br, &actualLen); @@ -4018,7 +4018,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); if (comp) { - tfree(outputBuf); + taosMemoryFreeClear(outputBuf); } } diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index bede8b80fdeb738792eeb3f1685d774287099e3d..814aa48b556f40a9c5013c19b60a218f3159a133 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -52,10 +52,10 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { } else if (pNode->nodeType == TEXPR_VALUE_NODE) { taosVariantDestroy(pNode->pVal); } else if (pNode->nodeType == TEXPR_COL_NODE) { - tfree(pNode->pSchema); + taosMemoryFreeClear(pNode->pSchema); } - free(pNode); + taosMemoryFree(pNode); } static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { @@ -80,12 +80,12 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { assert((*pExpr)->_node.pRight == NULL); } else if (type == TEXPR_VALUE_NODE) { taosVariantDestroy((*pExpr)->pVal); - free((*pExpr)->pVal); + taosMemoryFree((*pExpr)->pVal); } else if (type == TEXPR_COL_NODE) { - free((*pExpr)->pSchema); + taosMemoryFree((*pExpr)->pSchema); } - free(*pExpr); + taosMemoryFree(*pExpr); *pExpr = NULL; } @@ -154,7 +154,7 @@ void exprTreeToBinary(SBufferWriter* bw, tExprNode* expr) { // TODO: these three functions should be made global static void* exception_calloc(size_t nmemb, size_t size) { - void* p = calloc(nmemb, size); + void* p = taosMemoryCalloc(nmemb, size); if (p == NULL) { THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -162,7 +162,7 @@ static void* exception_calloc(size_t nmemb, size_t size) { } static void* exception_malloc(size_t size) { - void* p = malloc(size); + void* p = taosMemoryMalloc(size); if (p == NULL) { THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -195,7 +195,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) { pVal->nType = tbufReadUint32(br); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { tbufReadToBuffer(br, &pVal->nLen, sizeof(pVal->nLen)); - pVal->pz = calloc(1, pVal->nLen + 1); + pVal->pz = taosMemoryCalloc(1, pVal->nLen + 1); tbufReadToBuffer(br, pVal->pz, pVal->nLen); } else { pVal->i = tbufReadInt64(br); @@ -377,7 +377,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t bufLen = 128; } - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + char *tmp = taosMemoryCalloc(1, bufLen * TSDB_NCHAR_SIZE); for (int32_t i = 0; i < sz; i++) { switch (sType) { @@ -440,7 +440,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); if (bufLen < t) { - tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); + tmp = taosMemoryRealloc(tmp, t * TSDB_NCHAR_SIZE); bufLen = (int32_t)t; } @@ -530,7 +530,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t err_ret: taosVariantDestroy(&tmpVar); taosHashCleanup(pObj); - tfree(tmp); + taosMemoryFreeClear(tmp); } tExprNode* exprdup(tExprNode* pNode) { @@ -538,7 +538,7 @@ tExprNode* exprdup(tExprNode* pNode) { return NULL; } - tExprNode* pCloned = calloc(1, sizeof(tExprNode)); + tExprNode* pCloned = taosMemoryCalloc(1, sizeof(tExprNode)); if (pNode->nodeType == TEXPR_BINARYEXPR_NODE) { tExprNode* pLeft = exprdup(pNode->_node.pLeft); tExprNode* pRight = exprdup(pNode->_node.pRight); @@ -547,17 +547,17 @@ tExprNode* exprdup(tExprNode* pNode) { pCloned->_node.pRight = pRight; pCloned->_node.optr = pNode->_node.optr; } else if (pNode->nodeType == TEXPR_VALUE_NODE) { - pCloned->pVal = calloc(1, sizeof(SVariant)); + pCloned->pVal = taosMemoryCalloc(1, sizeof(SVariant)); taosVariantAssign(pCloned->pVal, pNode->pVal); } else if (pNode->nodeType == TEXPR_COL_NODE) { - pCloned->pSchema = calloc(1, sizeof(SSchema)); + pCloned->pSchema = taosMemoryCalloc(1, sizeof(SSchema)); *pCloned->pSchema = *pNode->pSchema; } else if (pNode->nodeType == TEXPR_FUNCTION_NODE) { strcpy(pCloned->_function.functionName, pNode->_function.functionName); int32_t num = pNode->_function.num; pCloned->_function.num = num; - pCloned->_function.pChild = calloc(num, POINTER_BYTES); + pCloned->_function.pChild = taosMemoryCalloc(num, POINTER_BYTES); for(int32_t i = 0; i < num; ++i) { pCloned->_function.pChild[i] = exprdup(pNode->_function.pChild[i]); } diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index 46d82aa6fbbf1854eeb8415fe21340fd5d16dad5..1f7e83286b3c912db211d9e8b588e12dad9742b8 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -149,7 +149,7 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { return; } - *next = calloc(1, pFillInfo->rowSize); + *next = taosMemoryCalloc(1, pFillInfo->rowSize); for (int i = 1; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes); @@ -258,7 +258,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { /* the raw data block is exhausted, next value does not exists */ if (pFillInfo->index >= pFillInfo->numOfRows) { - tfree(*next); + taosMemoryFreeClear(*next); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; @@ -314,7 +314,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t pSchema->type = pColInfo->col.type; pSchema->bytes = pColInfo->col.bytes; - pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); + pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pColInfo->col.bytes); pColInfo->tagIndex = k; k += 1; @@ -342,12 +342,12 @@ static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - struct SFillColInfo* pCol, void* handle) { + struct SFillColInfo* pCol, const char* id) { if (fillType == TSDB_FILL_NONE) { return NULL; } - SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); + SFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SFillInfo)); taosResetFillInfo(pFillInfo, skey); pFillInfo->order = order; @@ -357,17 +357,17 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag pFillInfo->numOfCols = numOfCols; pFillInfo->precision = precision; pFillInfo->alloc = capacity; - pFillInfo->handle = handle; + pFillInfo->id = id; pFillInfo->interval.interval = slidingTime; pFillInfo->interval.intervalUnit = slidingUnit; pFillInfo->interval.sliding = slidingTime; pFillInfo->interval.slidingUnit = slidingUnit; - pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); + pFillInfo->pData = taosMemoryMalloc(POINTER_BYTES * numOfCols); // if (numOfTags > 0) { - pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo)); + pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); for (int32_t i = 0; i < numOfCols; ++i) { pFillInfo->pTags[i].col.colId = -2; // TODO } @@ -394,19 +394,19 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { return NULL; } - tfree(pFillInfo->prevValues); - tfree(pFillInfo->nextValues); + taosMemoryFreeClear(pFillInfo->prevValues); + taosMemoryFreeClear(pFillInfo->nextValues); for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { - tfree(pFillInfo->pTags[i].tagVal); + taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); } - tfree(pFillInfo->pTags); + taosMemoryFreeClear(pFillInfo->pTags); - tfree(pFillInfo->pData); - tfree(pFillInfo->pFillCol); + taosMemoryFreeClear(pFillInfo->pData); + taosMemoryFreeClear(pFillInfo->pFillCol); - tfree(pFillInfo); + taosMemoryFreeClear(pFillInfo); return NULL; } @@ -530,7 +530,7 @@ int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal) { int32_t offset = 0; - struct SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); + struct SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; } diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index 49799aef7aa930d39e0e1c3a29aff2b399d6e6c7..2e60498aba4d339edb5346107c44fac109d54761 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -34,11 +34,11 @@ static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { /* need one redundant slot */ - SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); + SHistogramInfo* pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); #if !defined(USE_ARRAYLIST) pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - SInsertSupporter* pss = malloc(sizeof(SInsertSupporter)); + SInsertSupporter* pss = taosMemoryMalloc(sizeof(SInsertSupporter)); pss->numOfEntries = pHisto->maxEntries; pss->pSkipList = pHisto->pList; @@ -96,7 +96,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } #else tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); - SHistBin* entry = calloc(1, sizeof(SHistBin)); + SHistBin* entry = taosMemoryCalloc(1, sizeof(SHistBin)); entry->val = val; tSkipListNode* pResNode = SSkipListPut((*pHisto)->pList, entry, &key, 0); @@ -352,7 +352,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto) { return; } - free(*pHisto); + taosMemoryFree(*pHisto); *pHisto = NULL; } @@ -417,7 +417,7 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { #if defined(USE_ARRAYLIST) - double* pVal = malloc(num * sizeof(double)); + double* pVal = taosMemoryMalloc(num * sizeof(double)); for (int32_t i = 0; i < num; ++i) { double numOfElem = (ratio[i] / 100) * pHisto->numOfElems; @@ -463,7 +463,7 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { } } #else - double* pVal = malloc(num * sizeof(double)); + double* pVal = taosMemoryMalloc(num * sizeof(double)); for (int32_t i = 0; i < num; ++i) { double numOfElem = ratio[i] * pHisto->numOfElems; @@ -535,7 +535,7 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 return pResHistogram; } - SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); + SHistBin* pHistoBins = taosMemoryCalloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); int32_t i = 0, j = 0, k = 0; while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { @@ -573,6 +573,6 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 pResHistogram->numOfEntries = k; memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); - free(pHistoBins); + taosMemoryFree(pHistoBins); return pResHistogram; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 06c58430a4ea44838d758a3fe56c2f428c8b89ff..dd57024624f09d807996309c34e3990a39fdab46 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -29,7 +29,7 @@ int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { } static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { - SFilePage *buffer = (SFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); + SFilePage *buffer = (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); @@ -216,7 +216,7 @@ static void resetSlotInfo(tMemBucket* pBucket) { } tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) { - tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket)); + tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); if (pBucket == NULL) { return NULL; } @@ -233,7 +233,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { // qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); - free(pBucket); + taosMemoryFree(pBucket); return NULL; } @@ -243,13 +243,13 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, pBucket->hashFunc = getHashFunc(pBucket->type); if (pBucket->hashFunc == NULL) { // qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); - free(pBucket); + taosMemoryFree(pBucket); return NULL; } - pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); + pBucket->pSlots = (tMemBucketSlot *)taosMemoryCalloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); if (pBucket->pSlots == NULL) { - free(pBucket); + taosMemoryFree(pBucket); return NULL; } @@ -271,8 +271,8 @@ void tMemBucketDestroy(tMemBucket *pBucket) { } destroyDiskbasedBuf(pBucket->pBuffer); - tfree(pBucket->pSlots); - tfree(pBucket); + taosMemoryFreeClear(pBucket->pSlots); + taosMemoryFreeClear(pBucket); } void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { @@ -449,7 +449,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal); double val = (1 - fraction) * td + fraction * nd; - tfree(buffer); + taosMemoryFreeClear(buffer); return val; } else { // incur a second round bucket split diff --git a/source/libs/function/src/tscript.c b/source/libs/function/src/tscript.c index 7c07b0b7831e685c88ba861b00f27e19f094161d..93e8ecf5cfbc7aa67ed7bd12e404d850fa3eb4f4 100644 --- a/source/libs/function/src/tscript.c +++ b/source/libs/function/src/tscript.c @@ -195,7 +195,7 @@ void taosLoadScriptDestroy(void *pInit) { } ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) { - ScriptCtx *pCtx = (ScriptCtx *)calloc(1, sizeof(ScriptCtx)); + ScriptCtx *pCtx = (ScriptCtx *)taosMemoryCalloc(1, sizeof(ScriptCtx)); pCtx->state = SCRIPT_STATE_INIT; pCtx->pEnv = getScriptEnvFromPool(); // pCtx->resType = resType; @@ -229,7 +229,7 @@ ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) { void destroyScriptCtx(void *pCtx) { if (pCtx == NULL) return; addScriptEnvToPool(((ScriptCtx *)pCtx)->pEnv); - free(pCtx); + taosMemoryFree(pCtx); } void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes) { @@ -332,12 +332,12 @@ void destroyLuaEnv(lua_State *lua) { int32_t scriptEnvPoolInit() { const int size = 10; // configure or not - pool = malloc(sizeof(ScriptEnvPool)); - pthread_mutex_init(&pool->mutex, NULL); + pool = taosMemoryMalloc(sizeof(ScriptEnvPool)); + taosThreadMutexInit(&pool->mutex, NULL); pool->scriptEnvs = tdListNew(sizeof(ScriptEnv *)); for (int i = 0; i < size; i++) { - ScriptEnv *env = malloc(sizeof(ScriptEnv)); + ScriptEnv *env = taosMemoryMalloc(sizeof(ScriptEnv)); env->funcId = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);; env->lua_state = createLuaEnv(); tdListAppend(pool->scriptEnvs, (void *)(&env)); @@ -359,22 +359,22 @@ void scriptEnvPoolCleanup() { listNodeFree(pNode); } tdListFree(pool->scriptEnvs); - pthread_mutex_destroy(&pool->mutex); - free(pool); + taosThreadMutexDestroy(&pool->mutex); + taosMemoryFree(pool); } void destroyScriptEnv(ScriptEnv *pEnv) { destroyLuaEnv(pEnv->lua_state); taosHashCleanup(pEnv->funcId); - free(pEnv); + taosMemoryFree(pEnv); } ScriptEnv* getScriptEnvFromPool() { ScriptEnv *pEnv = NULL; - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); if (pool->cSize <= 0) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return NULL; } SListNode *pNode = tdListPopHead(pool->scriptEnvs); @@ -384,7 +384,7 @@ ScriptEnv* getScriptEnvFromPool() { } pool->cSize--; - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return pEnv; } @@ -392,11 +392,11 @@ void addScriptEnvToPool(ScriptEnv *pEnv) { if (pEnv == NULL) { return; } - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); lua_settop(pEnv->lua_state, 0); tdListAppend(pool->scriptEnvs, (void *)(&pEnv)); pool->cSize++; - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); } bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index b65e637a57496486919f0114dd633077f337d8d1..a1030f6c21d5e42552515fd0e6af372e95d1a3f1 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1,195 +1,925 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "uv.h" +#include "os.h" +#include "tlog.h" #include "tudf.h" +#include "tudfInt.h" -static char* getUdfFuncName(char* funcname, char* name, int type) { - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - strcpy(funcname, name); +//TODO: when startup, set thread poll size. add it to cfg +//TODO: udfd restart when exist or aborts +//TODO: network error processing. +//TODO: add unit test +//TODO: add lua support +void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); + +enum { + UV_TASK_CONNECT = 0, + UV_TASK_REQ_RSP = 1, + UV_TASK_DISCONNECT = 2 +}; + +typedef struct SUdfUvSession { + int64_t severHandle; + uv_pipe_t *udfSvcPipe; +} SUdfUvSession; + +typedef struct SClientUvTaskNode { + int8_t type; + int errCode; + + uv_pipe_t *pipe; + + int64_t seqNum; + uv_buf_t reqBuf; + + uv_sem_t taskSem; + uv_buf_t rspBuf; + + struct SClientUvTaskNode *prev; + struct SClientUvTaskNode *next; +} SClientUvTaskNode; + +typedef struct SClientUdfTask { + int8_t type; + + SUdfUvSession *session; + + int32_t errCode; + + union { + struct { + SUdfSetupRequest req; + SUdfSetupResponse rsp; + } _setup; + struct { + SUdfCallRequest req; + SUdfCallResponse rsp; + } _call; + struct { + SUdfTeardownRequest req; + SUdfTeardownResponse rsp; + } _teardown; + }; + + +} SClientUdfTask; + +typedef struct SClientConnBuf { + char *buf; + int32_t len; + int32_t cap; + int32_t total; +} SClientConnBuf; + +typedef struct SClientUvConn { + uv_pipe_t *pipe; + SClientUvTaskNode taskQueue; + SClientConnBuf readBuf; +} SClientUvConn; + +uv_process_t gUdfdProcess; + +uv_barrier_t gUdfInitBarrier; + +uv_loop_t gUdfdLoop; +uv_thread_t gUdfLoopThread; +uv_async_t gUdfLoopTaskAync; + +uv_async_t gUdfLoopStopAsync; + +uv_mutex_t gUdfTaskQueueMutex; +int64_t gUdfTaskSeqNum = 0; + +//double circular linked list +typedef SClientUvTaskNode *SClientUvTaskQueue; +SClientUvTaskNode gUdfQueueNode; +SClientUvTaskQueue gUdfTaskQueue = &gUdfQueueNode; + +//add SClientUvTaskNode task that close conn + + + +void udfTaskQueueInit(SClientUvTaskQueue q) { + q->next = q; + q->prev = q; +} + +bool udfTaskQueueIsEmpty(SClientUvTaskQueue q) { + return q == q->next; +} + +void udfTaskQueueInsertTail(SClientUvTaskQueue q, SClientUvTaskNode *e) { + e->next = q; + e->prev = q->prev; + e->prev->next = e; + q->prev = e; +} + +void udfTaskQueueInsertTaskAtHead(SClientUvTaskQueue q, SClientUvTaskNode *e) { + e->next = q->next; + e->prev = q; + q->next->prev = e; + q->next = e; +} + +void udfTaskQueueRemoveTask(SClientUvTaskNode *e) { + e->prev->next = e->next; + e->next->prev = e->prev; +} + +void udfTaskQueueSplit(SClientUvTaskQueue q, SClientUvTaskNode *from, SClientUvTaskQueue n) { + n->prev = q->prev; + n->prev->next = n; + n->next = from; + q->prev = from->prev; + q->prev->next = q; + from->prev = n; +} + +SClientUvTaskNode *udfTaskQueueHeadTask(SClientUvTaskQueue q) { + return q->next; +} + +SClientUvTaskNode *udfTaskQueueTailTask(SClientUvTaskQueue q) { + return q->prev; +} + +SClientUvTaskNode *udfTaskQueueNext(SClientUvTaskNode *e) { + return e->next; +} + +void udfTaskQueueMove(SClientUvTaskQueue q, SClientUvTaskQueue n) { + if (udfTaskQueueIsEmpty(q)) { + udfTaskQueueInit(n); + } else { + SClientUvTaskNode *h = udfTaskQueueHeadTask(q); + udfTaskQueueSplit(q, h, n); + } +} + + +int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { + debugPrint("%s", "encoding request"); + + int len = sizeof(SUdfRequest) - sizeof(void *); + switch (request->type) { + case UDF_TASK_SETUP: { + SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); + len += sizeof(SUdfSetupRequest) - 1 * sizeof(char *) + setup->pathSize; break; - case TSDB_UDF_FUNC_INIT: - sprintf(funcname, "%s_init", name); + } + case UDF_TASK_CALL: { + SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); + len += sizeof(SUdfCallRequest) - 2 * sizeof(char *) + call->inputBytes + call->stateBytes; break; - case TSDB_UDF_FUNC_FINALIZE: - sprintf(funcname, "%s_finalize", name); + } + case UDF_TASK_TEARDOWN: { + SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); + len += sizeof(SUdfTeardownRequest); break; - case TSDB_UDF_FUNC_MERGE: - sprintf(funcname, "%s_merge", name); + } + default: break; - case TSDB_UDF_FUNC_DESTROY: - sprintf(funcname, "%s_destroy", name); + } + + char *bufBegin = taosMemoryMalloc(len); + char *buf = bufBegin; + + //skip msgLen first + buf += sizeof(int32_t); + + *(int64_t *) buf = request->seqNum; + buf += sizeof(int64_t); + *(int8_t *) buf = request->type; + buf += sizeof(int8_t); + + switch (request->type) { + case UDF_TASK_SETUP: { + SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); + memcpy(buf, setup->udfName, 16); + buf += 16; + *(int8_t *) buf = setup->scriptType; + buf += sizeof(int8_t); + *(int8_t *) buf = setup->udfType; + buf += sizeof(int8_t); + *(int16_t *) buf = setup->pathSize; + buf += sizeof(int16_t); + memcpy(buf, setup->path, setup->pathSize); + buf += setup->pathSize; + break; + } + + case UDF_TASK_CALL: { + SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); + *(int64_t *) buf = call->udfHandle; + buf += sizeof(int64_t); + *(int8_t *) buf = call->step; + buf += sizeof(int8_t); + *(int32_t *) buf = call->inputBytes; + buf += sizeof(int32_t); + memcpy(buf, call->input, call->inputBytes); + buf += call->inputBytes; + *(int32_t *) buf = call->stateBytes; + buf += sizeof(int32_t); + memcpy(buf, call->state, call->stateBytes); + buf += call->stateBytes; break; + } + + case UDF_TASK_TEARDOWN: { + SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); + *(int64_t *) buf = teardown->udfHandle; + buf += sizeof(int64_t); + break; + } default: - assert(0); break; } - return funcname; + request->msgLen = buf - bufBegin; + *(int32_t *) bufBegin = request->msgLen; + *pBuf = bufBegin; + *pBufLen = request->msgLen; + return 0; } -#if 0 -int32_t initUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return TSDB_CODE_SUCCESS; +int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { + debugPrint("%s", "decoding request"); + if (*(int32_t *) bufMsg != bufLen) { + debugPrint("%s", "decoding request error"); + return -1; } - ////qError("script len: %d", pUdfInfo->contLen); - if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) { - pUdfInfo->isScript = 1; - pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes); - if (pUdfInfo->pScriptCtx == NULL) { - return TSDB_CODE_QRY_SYS_ERROR; + char *buf = bufMsg; + SUdfRequest *request = taosMemoryMalloc(sizeof(SUdfRequest)); + request->subReq = NULL; + request->msgLen = *(int32_t *) (buf); + buf += sizeof(int32_t); + request->seqNum = *(int64_t *) (buf); + buf += sizeof(int64_t); + request->type = *(int8_t *) (buf); + buf += sizeof(int8_t); + + switch (request->type) { + case UDF_TASK_SETUP: { + SUdfSetupRequest *setup = taosMemoryMalloc(sizeof(SUdfSetupRequest)); + + memcpy(setup->udfName, buf, 16); + buf += 16; + setup->scriptType = *(int8_t *) buf; + buf += sizeof(int8_t); + setup->udfType = *(int8_t *) buf; + buf += sizeof(int8_t); + setup->pathSize = *(int16_t *) buf; + buf += sizeof(int16_t); + setup->path = buf; + buf += setup->pathSize; + + request->subReq = setup; + break; } - tfree(pUdfInfo->content); - - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit; - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL - || (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_QRY_SYS_ERROR; + case UDF_TASK_CALL: { + SUdfCallRequest *call = taosMemoryMalloc(sizeof(SUdfCallRequest)); + + call->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + call->step = *(int8_t *) buf; + buf += sizeof(int8_t); + call->inputBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + call->input = buf; + buf += call->inputBytes; + call->stateBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + call->state = buf; + buf += call->stateBytes; + + request->subReq = call; + break; } - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal; + case UDF_TASK_TEARDOWN: { + SUdfTeardownRequest *teardown = taosMemoryMalloc(sizeof(SUdfTeardownRequest)); + + teardown->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); - if (pUdfInfo->funcType == FUNCTION_TYPE_AGG) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize; - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge; + request->subReq = teardown; } - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy; - } else { - char path[PATH_MAX] = {0}; - taosGetTmpfilePath("script", path, tsTempDir); + } + if (buf - bufMsg != bufLen) { + debugPrint("%s", "decode request error"); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + return -1; + } + *pRequest = request; + return 0; +} + +int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { + debugPrint("%s", "encoding response"); + + int32_t len = sizeof(SUdfResponse) - sizeof(void *); + + switch (response->type) { + case UDF_TASK_SETUP: { + len += sizeof(SUdfSetupResponse); + break; + } + case UDF_TASK_CALL: { + SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); + len += sizeof(SUdfCallResponse) - 2 * sizeof(char *) + + callResp->outputBytes + callResp->newStateBytes; + break; + } + case UDF_TASK_TEARDOWN: { + len += sizeof(SUdfTeardownResponse); + break; + } + } - FILE* file = fopen(path, "w+"); + char *bufBegin = taosMemoryMalloc(len); + char *buf = bufBegin; - // TODO check for failure of flush to disk - /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); - fclose(file); - tfree(pUdfInfo->content); + //skip msgLen + buf += sizeof(int32_t); - pUdfInfo->path = strdup(path); + *(int64_t *) buf = response->seqNum; + buf += sizeof(int64_t); + *(int8_t *) buf = response->type; + buf += sizeof(int8_t); + *(int32_t *) buf = response->code; + buf += sizeof(int32_t); - pUdfInfo->handle = taosLoadDll(path); - if (NULL == pUdfInfo->handle) { - return TSDB_CODE_QRY_SYS_ERROR; + switch (response->type) { + case UDF_TASK_SETUP: { + SUdfSetupResponse *setupResp = (SUdfSetupResponse *) (response->subRsp); + *(int64_t *) buf = setupResp->udfHandle; + buf += sizeof(int64_t); + break; + } + case UDF_TASK_CALL: { + SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); + *(int32_t *) buf = callResp->outputBytes; + buf += sizeof(int32_t); + memcpy(buf, callResp->output, callResp->outputBytes); + buf += callResp->outputBytes; + + *(int32_t *) buf = callResp->newStateBytes; + buf += sizeof(int32_t); + memcpy(buf, callResp->newState, callResp->newStateBytes); + buf += callResp->newStateBytes; + break; } + case UDF_TASK_TEARDOWN: { + SUdfTeardownResponse *teardownResp = (SUdfTeardownResponse *) (response->subRsp); + break; + } + default: + break; + } + response->msgLen = buf - bufBegin; + *(int32_t *) bufBegin = response->msgLen; + *pBuf = bufBegin; + *pBufLen = response->msgLen; + return 0; +} - char funcname[FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); - if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { - return TSDB_CODE_QRY_SYS_ERROR; +int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { + debugPrint("%s", "decoding response"); + + if (*(int32_t *) bufMsg != bufLen) { + debugPrint("%s", "can not decode response"); + return -1; + } + char *buf = bufMsg; + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); + rsp->msgLen = *(int32_t *) buf; + buf += sizeof(int32_t); + rsp->seqNum = *(int64_t *) buf; + buf += sizeof(int64_t); + rsp->type = *(int8_t *) buf; + buf += sizeof(int8_t); + rsp->code = *(int32_t *) buf; + buf += sizeof(int32_t); + + switch (rsp->type) { + case UDF_TASK_SETUP: { + SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) taosMemoryMalloc(sizeof(SUdfSetupResponse)); + setupRsp->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + rsp->subRsp = (char *) setupRsp; + break; } + case UDF_TASK_CALL: { + SUdfCallResponse *callRsp = (SUdfCallResponse *) taosMemoryMalloc(sizeof(SUdfCallResponse)); + callRsp->outputBytes = *(int32_t *) buf; + buf += sizeof(int32_t); - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); + callRsp->output = buf; + buf += callRsp->outputBytes; - if (pUdfInfo->funcType == FUNCTION_TYPE_AGG) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + callRsp->newStateBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + + callRsp->newState = buf; + buf += callRsp->newStateBytes; + + rsp->subRsp = callRsp; + break; } + case UDF_TASK_TEARDOWN: { + SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) taosMemoryMalloc(sizeof(SUdfTeardownResponse)); + rsp->subRsp = teardownRsp; + break; + } + default: + break; + } + if (buf - bufMsg != bufLen) { + debugPrint("%s", "can not decode response"); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + return -1; + } + *pResponse = rsp; + return 0; +} + +void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { + debugPrint("Process exited with status %" PRId64 ", signal %d", exit_status, term_signal); + uv_close((uv_handle_t *) req, NULL); + //TODO: restart the udfd process +} - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); +void onUdfcPipeClose(uv_handle_t *handle) { + SClientUvConn *conn = handle->data; + if (!udfTaskQueueIsEmpty(&conn->taskQueue)) { + SClientUvTaskNode *task = udfTaskQueueHeadTask(&conn->taskQueue); + task->errCode = 0; + uv_sem_post(&task->taskSem); + } + + taosMemoryFree(conn->readBuf.buf); + taosMemoryFree(conn); + taosMemoryFree((uv_pipe_t *) handle); + +} + +int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { + debugPrint("%s", "get uv task result"); + if (uvTask->type == UV_TASK_REQ_RSP) { + if (uvTask->rspBuf.base != NULL) { + SUdfResponse *rsp; + decodeResponse(uvTask->rspBuf.base, uvTask->rspBuf.len, &rsp); + task->errCode = rsp->code; + + switch (task->type) { + case UDF_TASK_SETUP: { + //TODO: copy or not + task->_setup.rsp = *(SUdfSetupResponse *) (rsp->subRsp); + break; + } + case UDF_TASK_CALL: { + task->_call.rsp = *(SUdfCallResponse *) (rsp->subRsp); + //TODO: copy or not + break; + } + case UDF_TASK_TEARDOWN: { + task->_teardown.rsp = *(SUdfTeardownResponse *) (rsp->subRsp); + //TODO: copy or not? + break; + } + default: { + break; + } + } + + // TODO: the call buffer is setup and freed by udf invocation + taosMemoryFree(uvTask->rspBuf.base); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + } else { + task->errCode = uvTask->errCode; + } + } else if (uvTask->type == UV_TASK_CONNECT) { + task->errCode = uvTask->errCode; + } else if (uvTask->type == UV_TASK_DISCONNECT) { + task->errCode = uvTask->errCode; + } + return 0; +} - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) { - return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init); +void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { + debugPrint("%s", "client allocate buffer to receive from pipe"); + SClientUvConn *conn = handle->data; + SClientConnBuf *connBuf = &conn->readBuf; + + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (connBuf->cap == 0) { + connBuf->buf = taosMemoryMalloc(msgHeadSize); + if (connBuf->buf) { + connBuf->len = 0; + connBuf->cap = msgHeadSize; + connBuf->total = -1; + + buf->base = connBuf->buf; + buf->len = connBuf->cap; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } else { + connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; + void *resultBuf = taosMemoryRealloc(connBuf->buf, connBuf->cap); + if (resultBuf) { + connBuf->buf = resultBuf; + buf->base = connBuf->buf + connBuf->len; + buf->len = connBuf->cap - connBuf->len; + } else { + //TODO: log error free connBuf->buf + buf->base = NULL; + buf->len = 0; } } - return TSDB_CODE_SUCCESS; + debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); + } -void destroyUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return; +bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { + if (connBuf->total == -1 && connBuf->len >= sizeof(int32_t)) { + connBuf->total = *(int32_t *) (connBuf->buf); } + if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + return true; + } + return false; +} + +void udfcUvHandleRsp(SClientUvConn *conn) { + SClientConnBuf *connBuf = &conn->readBuf; + int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum - if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { - if (pUdfInfo->isScript) { - (*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx); - tfree(pUdfInfo->content); - }else{ - (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); + if (udfTaskQueueIsEmpty(&conn->taskQueue)) { + //LOG error + return; + } + bool found = false; + SClientUvTaskNode *taskFound = NULL; + SClientUvTaskNode *task = udfTaskQueueNext(&conn->taskQueue); + while (task != &conn->taskQueue) { + if (task->seqNum == seqNum) { + if (found == false) { + found = true; + taskFound = task; + } else { + //LOG error; + continue; + } } + task = udfTaskQueueNext(task); } - tfree(pUdfInfo->name); + if (taskFound) { + taskFound->rspBuf = uv_buf_init(connBuf->buf, connBuf->len); + udfTaskQueueRemoveTask(taskFound); + uv_sem_post(&taskFound->taskSem); + } else { + //LOG error + } + connBuf->buf = NULL; + connBuf->total = -1; + connBuf->len = 0; + connBuf->cap = 0; +} + +void udfcUvHandleError(SClientUvConn *conn) { + uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); +} + +void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + debugPrint("%s, nread: %zd", "client read from pipe", nread); + if (nread == 0) return; + + SClientUvConn *conn = client->data; + SClientConnBuf *connBuf = &conn->readBuf; + if (nread > 0) { + connBuf->len += nread; + if (isUdfcUvMsgComplete(connBuf)) { + udfcUvHandleRsp(conn); + } - if (pUdfInfo->path) { - unlink(pUdfInfo->path); + } + if (nread < 0) { + debugPrint("\tclient read error: %s", uv_strerror(nread)); + if (nread == UV_EOF) { + //TODO: + } + udfcUvHandleError(conn); } - tfree(pUdfInfo->path); - tfree(pUdfInfo->content); - taosCloseDll(pUdfInfo->handle); - tfree(pUdfInfo); } -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type) { - int32_t output = 0; +void onUdfClientWrite(uv_write_t *write, int status) { + debugPrint("%s", "after writing to pipe"); + SClientUvTaskNode *uvTask = write->data; + if (status == 0) { + uv_pipe_t *pipe = uvTask->pipe; + SClientUvConn *conn = pipe->data; + udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + } else { + //TODO Log error; + } + debugPrint("\tlength:%zu", uvTask->reqBuf.len); + taosMemoryFree(write); + taosMemoryFree(uvTask->reqBuf.base); +} - if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { - //qError("empty udf function, type:%d", type); - return; +void onUdfClientConnect(uv_connect_t *connect, int status) { + SClientUvTaskNode *uvTask = connect->data; + uvTask->errCode = status; + if (status != 0) { + //TODO: LOG error } + uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); + taosMemoryFree(connect); + uv_sem_post(&uvTask->taskSem); +} -// //qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); +int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { + SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode)); + uvTask->type = uvTaskType; + + if (uvTaskType == UV_TASK_CONNECT) { + } else if (uvTaskType == UV_TASK_REQ_RSP) { + uvTask->pipe = task->session->udfSvcPipe; + SUdfRequest request; + request.type = task->type; + request.seqNum = gUdfTaskSeqNum++; + + if (task->type == UDF_TASK_SETUP) { + request.subReq = &task->_setup.req; + request.type = UDF_TASK_SETUP; + } else if (task->type == UDF_TASK_CALL) { + request.subReq = &task->_call.req; + request.type = UDF_TASK_CALL; + } else if (task->type == UDF_TASK_TEARDOWN) { + request.subReq = &task->_teardown.req; + request.type = UDF_TASK_TEARDOWN; + } else { + //TODO log and return error + } + char *buf = NULL; + int32_t bufLen = 0; + encodeRequest(&buf, &bufLen, &request); + uvTask->reqBuf = uv_buf_init(buf, bufLen); + uvTask->seqNum = request.seqNum; + } else if (uvTaskType == UV_TASK_DISCONNECT) { + uvTask->pipe = task->session->udfSvcPipe; + } + uv_sem_init(&uvTask->taskSem, 0); - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - if (pUdfInfo->isScript) { - (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, - (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, - (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + *pUvTask = uvTask; + return 0; +} - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); +int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { + debugPrint("%s, %d", "queue uv task", uvTask->type); - (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, - pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes, &pUdfInfo->init); - } + uv_mutex_lock(&gUdfTaskQueueMutex); + udfTaskQueueInsertTail(gUdfTaskQueue, uvTask); + uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_async_send(&gUdfLoopTaskAync); - if (pUdfInfo->funcType == TSDB_FUNC_TYPE_AGGREGATE) { - pCtx->resultInfo->numOfRes = output; - } else { - pCtx->resultInfo->numOfRes += output; - } + uv_sem_wait(&uvTask->taskSem); + uv_sem_destroy(&uvTask->taskSem); - if (pCtx->resultInfo->numOfRes > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } + return 0; +} - break; +int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { + debugPrint("%s, type %d", "start uv task ", uvTask->type); + switch (uvTask->type) { + case UV_TASK_CONNECT: { + uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); + uv_pipe_init(&gUdfdLoop, pipe, 0); + uvTask->pipe = pipe; - case TSDB_UDF_FUNC_MERGE: - if (pUdfInfo->isScript) { - (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); - } else { - (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); - } + SClientUvConn *conn = taosMemoryMalloc(sizeof(SClientUvConn)); + conn->pipe = pipe; + conn->readBuf.len = 0; + conn->readBuf.cap = 0; + conn->readBuf.buf = 0; + conn->readBuf.total = -1; + udfTaskQueueInit(&conn->taskQueue); - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } + pipe->data = conn; + + uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); + connReq->data = uvTask; + uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); + break; + } + case UV_TASK_REQ_RSP: { + uv_pipe_t *pipe = uvTask->pipe; + uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); + write->data = uvTask; + uv_write(write, (uv_stream_t *) pipe, &uvTask->reqBuf, 1, onUdfClientWrite); break; + } + case UV_TASK_DISCONNECT: { + SClientUvConn *conn = uvTask->pipe->data; + udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); + break; + } + default: { + break; + } + } - case TSDB_UDF_FUNC_FINALIZE: { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - if (pUdfInfo->isScript) { - (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); - } else { - (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); - } - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } + return 0; +} - break; - } +void udfClientAsyncCb(uv_async_t *async) { + SClientUvTaskNode node; + SClientUvTaskQueue q = &node; + udfTaskQueueInit(q); + + uv_mutex_lock(&gUdfTaskQueueMutex); + udfTaskQueueMove(gUdfTaskQueue, q); + uv_mutex_unlock(&gUdfTaskQueueMutex); + + while (!udfTaskQueueIsEmpty(q)) { + SClientUvTaskNode *task = udfTaskQueueHeadTask(q); + udfTaskQueueRemoveTask(task); + startUvUdfTask(task); + } + +} + +void udfStopAsyncCb(uv_async_t *async) { + uv_stop(&gUdfdLoop); + uv_loop_close(&gUdfdLoop); +} + +void startUdfd(void *argsThread) { + uv_loop_init(&gUdfdLoop); + + //TODO: path + uv_process_options_t options; + static char path[256] = {0}; + size_t cwdSize; + uv_cwd(path, &cwdSize); + strcat(path, "./udfd"); + char* args[2] = {path, NULL}; + options.args = args; + options.file = path; + options.exit_cb = onUdfdExit; + + int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); + if (err != 0) { + debugPrint("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); + } + + uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); + uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); + uv_mutex_init(&gUdfTaskQueueMutex); + udfTaskQueueInit(gUdfTaskQueue); + uv_barrier_wait(&gUdfInitBarrier); + uv_run(&gUdfdLoop, UV_RUN_DEFAULT); +} + +int32_t startUdfService() { + uv_barrier_init(&gUdfInitBarrier, 2); + uv_thread_create(&gUdfLoopThread, startUdfd, 0); + uv_barrier_wait(&gUdfInitBarrier); + return 0; +} + +int32_t stopUdfService() { + uv_barrier_destroy(&gUdfInitBarrier); + uv_process_kill(&gUdfdProcess, SIGINT); + uv_async_send(&gUdfLoopStopAsync); + uv_mutex_destroy(&gUdfTaskQueueMutex); + uv_thread_join(&gUdfLoopThread); + return 0; +} + +int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { + SClientUvTaskNode *uvTask = NULL; + + createUdfcUvTask(task, uvTaskType, &uvTask); + queueUvUdfTask(uvTask); + udfcGetUvTaskResponseResult(task, uvTask); + if (uvTaskType == UV_TASK_CONNECT) { + task->session->udfSvcPipe = uvTask->pipe; + } + taosMemoryFree(uvTask); + uvTask = NULL; + return task->errCode; +} + +int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { + debugPrint("%s", "client setup udf"); + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); + task->type = UDF_TASK_SETUP; + + SUdfSetupRequest *req = &task->_setup.req; + memcpy(req->udfName, udfInfo->udfName, 16); + req->path = udfInfo->path; + req->pathSize = strlen(req->path) + 1; + req->udfType = udfInfo->udfType; + req->scriptType = udfInfo->scriptType; + + int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); + if (errCode != 0) { + //TODO: log error + return -1; } + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + SUdfSetupResponse *rsp = &task->_setup.rsp; + task->session->severHandle = rsp->udfHandle; + *handle = task->session; + int32_t err = task->errCode; + taosMemoryFree(task); + return err; } -#endif \ No newline at end of file +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newState, + int32_t *newStateSize, SUdfDataBlock *output) { + debugPrint("%s", "client call udf"); + + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = (SUdfUvSession *) handle; + task->type = UDF_TASK_CALL; + + SUdfCallRequest *req = &task->_call.req; + + req->state = state; + req->stateBytes = stateSize; + req->inputBytes = input.size; + req->input = input.data; + req->udfHandle = task->session->severHandle; + req->step = step; + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + SUdfCallResponse *rsp = &task->_call.rsp; + *newState = rsp->newState; + *newStateSize = rsp->newStateBytes; + output->size = rsp->outputBytes; + output->data = rsp->output; + int32_t err = task->errCode; + taosMemoryFree(task); + return err; +} + +int32_t teardownUdf(UdfHandle handle) { + debugPrint("%s", "client teardown udf"); + + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = (SUdfUvSession *) handle; + task->type = UDF_TASK_TEARDOWN; + + SUdfTeardownRequest *req = &task->_teardown.req; + req->udfHandle = task->session->severHandle; + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + + SUdfTeardownResponse *rsp = &task->_teardown.rsp; + + int32_t err = task->errCode; + + udfcRunUvTask(task, UV_TASK_DISCONNECT); + + taosMemoryFree(task->session); + taosMemoryFree(task); + + return err; +} diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c new file mode 100644 index 0000000000000000000000000000000000000000..b473f060c099c1277e6c92752980e8c2e6a7bed6 --- /dev/null +++ b/source/libs/function/src/udfd.c @@ -0,0 +1,356 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "uv.h" +#include "os.h" +#include "tlog.h" + +#include "tudf.h" +#include "tudfInt.h" + + +static uv_loop_t *loop; + +typedef struct SUdfdUvConn { + uv_stream_t *client; + char *inputBuf; + int32_t inputLen; + int32_t inputCap; + int32_t inputTotal; +} SUdfdUvConn; + +typedef struct SUvUdfWork { + uv_stream_t *client; + uv_buf_t input; + uv_buf_t output; +} SUvUdfWork; + +typedef struct SUdf { + int32_t refCount; + + char name[16]; + int8_t type; + + uv_lib_t lib; + TUdfFunc normalFunc; +} SUdf; + +//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix +//TODO: add private udf structure. +typedef struct SUdfHandle { + SUdf *udf; +} SUdfHandle; + + +void udfdProcessRequest(uv_work_t *req) { + SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data); + SUdfRequest *request = NULL; + decodeRequest(uvUdf->input.base, uvUdf->input.len, &request); + + switch (request->type) { + case UDF_TASK_SETUP: { + debugPrint("%s", "process setup request"); + SUdf *udf = taosMemoryMalloc(sizeof(SUdf)); + udf->refCount = 0; + SUdfSetupRequest *setup = request->subReq; + strcpy(udf->name, setup->udfName); + int err = uv_dlopen(setup->path, &udf->lib); + if (err != 0) { + debugPrint("can not load library %s. error: %s", setup->path, uv_strerror(err)); + //TODO set error + } + + char normalFuncName[32] = {0}; + strcpy(normalFuncName, setup->udfName); + //TODO error, + //TODO find all functions normal, init, destroy, normal, merge, finalize + uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc)); + + SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); + handle->udf = udf; + udf->refCount++; + //TODO: allocate private structure and call init function and set it to handle + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfSetupResponse *subRsp = taosMemoryMalloc(sizeof(SUdfSetupResponse)); + subRsp->udfHandle = (int64_t) (handle); + rsp->subRsp = subRsp; + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + + uvUdf->output = uv_buf_init(buf, len); + + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); + break; + } + + case UDF_TASK_CALL: { + debugPrint("%s", "process call request"); + SUdfCallRequest *call = request->subReq; + SUdfHandle *handle = (SUdfHandle *) (call->udfHandle); + SUdf *udf = handle->udf; + char *newState; + int32_t newStateSize; + SUdfDataBlock input = {.data = call->input, .size= call->inputBytes}; + SUdfDataBlock output; + //TODO: call different functions according to the step + udf->normalFunc(call->step, call->state, call->stateBytes, input, &newState, &newStateSize, &output); + + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfCallResponse *subRsp = taosMemoryMalloc(sizeof(SUdfCallResponse)); + subRsp->outputBytes = output.size; + subRsp->output = output.data; + subRsp->newStateBytes = newStateSize; + subRsp->newState = newState; + rsp->subRsp = subRsp; + + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + uvUdf->output = uv_buf_init(buf, len); + + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(newState); + taosMemoryFree(output.data); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); + break; + } + case UDF_TASK_TEARDOWN: { + debugPrint("%s", "process teardown request"); + + SUdfTeardownRequest *teardown = request->subReq; + SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle); + SUdf *udf = handle->udf; + udf->refCount--; + if (udf->refCount == 0) { + uv_dlclose(&udf->lib); + } + taosMemoryFree(udf); + //TODO: call destroy and free udf private + taosMemoryFree(handle); + + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfTeardownResponse *subRsp = taosMemoryMalloc(sizeof(SUdfTeardownResponse)); + rsp->subRsp = subRsp; + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + uvUdf->output = uv_buf_init(buf, len); + + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); + break; + } + default: { + break; + } + + } + +} + +void udfdOnWrite(uv_write_t *req, int status) { + debugPrint("%s", "after writing to pipe"); + if (status < 0) { + debugPrint("Write error %s", uv_err_name(status)); + } + SUvUdfWork *work = (SUvUdfWork *) req->data; + debugPrint("\tlength: %zu", work->output.len); + taosMemoryFree(work->output.base); + taosMemoryFree(work); + taosMemoryFree(req); +} + + +void udfdSendResponse(uv_work_t *work, int status) { + debugPrint("%s", "send response"); + SUvUdfWork *udfWork = (SUvUdfWork *) (work->data); + + uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); + write_req->data = udfWork; + uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); + + taosMemoryFree(work); +} + +void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { + debugPrint("%s", "allocate buffer for read"); + SUdfdUvConn *ctx = handle->data; + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (ctx->inputCap == 0) { + ctx->inputBuf = taosMemoryMalloc(msgHeadSize); + if (ctx->inputBuf) { + ctx->inputLen = 0; + ctx->inputCap = msgHeadSize; + ctx->inputTotal = -1; + + buf->base = ctx->inputBuf; + buf->len = ctx->inputCap; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } else { + ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; + void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); + if (inputBuf) { + ctx->inputBuf = inputBuf; + buf->base = ctx->inputBuf + ctx->inputLen; + buf->len = ctx->inputCap - ctx->inputLen; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } + debugPrint("\tinput buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal); + +} + +bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) { + if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) { + pipe->inputTotal = *(int32_t *) (pipe->inputBuf); + } + if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) { + return true; + } + return false; +} + +void udfdHandleRequest(SUdfdUvConn *conn) { + uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); + SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); + udfWork->client = conn->client; + udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); + conn->inputBuf = NULL; + conn->inputLen = 0; + conn->inputCap = 0; + conn->inputTotal = -1; + work->data = udfWork; + uv_queue_work(loop, work, udfdProcessRequest, udfdSendResponse); +} + +void udfdPipeCloseCb(uv_handle_t *pipe) { + SUdfdUvConn *conn = pipe->data; + taosMemoryFree(conn->client); + taosMemoryFree(conn->inputBuf); + taosMemoryFree(conn); +} + +void udfdUvHandleError(SUdfdUvConn *conn) { + uv_close((uv_handle_t *) conn->client, udfdPipeCloseCb); +} + +void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + debugPrint("%s, nread: %zd", "read from pipe", nread); + + if (nread == 0) return; + + SUdfdUvConn *conn = client->data; + + if (nread > 0) { + conn->inputLen += nread; + if (isUdfdUvMsgComplete(conn)) { + udfdHandleRequest(conn); + } else { + //log error or continue; + } + return; + } + + if (nread < 0) { + debugPrint("Read error %s", uv_err_name(nread)); + if (nread == UV_EOF) { + //TODO check more when close + } else { + } + udfdUvHandleError(conn); + } +} + +void udfdOnNewConnection(uv_stream_t *server, int status) { + debugPrint("%s", "on new connection"); + if (status < 0) { + // TODO + return; + } + + uv_pipe_t *client = (uv_pipe_t *) taosMemoryMalloc(sizeof(uv_pipe_t)); + uv_pipe_init(loop, client, 0); + if (uv_accept(server, (uv_stream_t *) client) == 0) { + SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); + ctx->client = (uv_stream_t *) client; + ctx->inputBuf = 0; + ctx->inputLen = 0; + ctx->inputCap = 0; + client->data = ctx; + ctx->client = (uv_stream_t *) client; + uv_read_start((uv_stream_t *) client, udfdAllocBuffer, udfdPipeRead); + } else { + uv_close((uv_handle_t *) client, NULL); + } +} + +void removeListeningPipe(int sig) { + uv_fs_t req; + uv_fs_unlink(loop, &req, "udf.sock", NULL); + exit(0); +} + +int main() { + debugPrint("libuv version: %x", UV_VERSION_HEX); + + loop = uv_default_loop(); + uv_fs_t req; + uv_fs_unlink(loop, &req, "udf.sock", NULL); + + uv_pipe_t server; + uv_pipe_init(loop, &server, 0); + + signal(SIGINT, removeListeningPipe); + + int r; + if ((r = uv_pipe_bind(&server, "udf.sock"))) { + debugPrint("Bind error %s\n", uv_err_name(r)); + removeListeningPipe(0); + return 1; + } + if ((r = uv_listen((uv_stream_t *) &server, 128, udfdOnNewConnection))) { + debugPrint("Listen error %s", uv_err_name(r)); + return 2; + } + uv_run(loop, UV_RUN_DEFAULT); + uv_loop_close(loop); +} diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c new file mode 100644 index 0000000000000000000000000000000000000000..bd742d23d07ba5ae1467dcfc3fb658c8cd9bc4ef --- /dev/null +++ b/source/libs/function/test/runUdf.c @@ -0,0 +1,46 @@ +#include +#include +#include + +#include "uv.h" +#include "os.h" +#include "tudf.h" + +int main(int argc, char *argv[]) { + startUdfService(); + uv_sleep(1000); + char path[256] = {0}; + size_t cwdSize = 256; + int err = uv_cwd(path, &cwdSize); + if (err != 0) { + fprintf(stderr, "err cwd: %s\n", uv_strerror(err)); + return err; + } + fprintf(stdout, "current working directory:%s\n", path); + strcat(path, "/libudf1.so"); + SUdfInfo udfInfo = {.udfName="udf1", .path=path}; + + UdfHandle handle; + setupUdf(&udfInfo, &handle); + + //char state[5000000] = "state"; + //char input[5000000] = "input"; + int dataSize = 500; + int callCount = 2; + if (argc > 1) dataSize = atoi(argv[1]); + if (argc > 2) callCount = atoi(argv[2]); + char *state = taosMemoryMalloc(dataSize); + char *input = taosMemoryMalloc(dataSize); + SUdfDataBlock blockInput = {.data = input, .size = dataSize}; + SUdfDataBlock blockOutput; + char* newState; + int32_t newStateSize; + for (int l = 0; l < callCount; ++l) { + callUdf(handle, 0, state, dataSize, blockInput, &newState, &newStateSize, &blockOutput); + } + taosMemoryFree(state); + taosMemoryFree(input); + teardownUdf(handle); + + stopUdfService(); +} diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c new file mode 100644 index 0000000000000000000000000000000000000000..dc88e8cf3e194532a34414b9362711141b0b8a9d --- /dev/null +++ b/source/libs/function/test/udf1.c @@ -0,0 +1,21 @@ +#include +#include +#include + +#include "os.h" +#include "tudf.h" + +void udf1(int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, + char **newState, int32_t *newStateSize, SUdfDataBlock *output) { + fprintf(stdout, "%s, step:%d\n", "udf function called", step); + char *newStateBuf = taosMemoryMalloc(stateSize); + memcpy(newStateBuf, state, stateSize); + *newState = newStateBuf; + *newStateSize = stateSize; + + char *outputBuf = taosMemoryMalloc(input.size); + memcpy(outputBuf, input.data, input.size); + output->data = outputBuf; + output->size = input.size; + return; +} diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 3d1d5356c24e23a3e89eedd59ca2a58b1e4d01ae..37318767c71e8558d284cccb05c6c5d77f48cbdf 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -58,7 +58,7 @@ struct SIndex { char* path; SIndexStat stat; - pthread_mutex_t mtx; + TdThreadMutex mtx; }; struct SIndexOpts { diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index a6ebcd6d6ffbf700c482e8094754ebe1c7e54742..086e75d99fd85887701daad11b88310c417c9bfe 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -43,8 +43,8 @@ typedef struct IndexCache { int8_t type; uint64_t suid; - pthread_mutex_t mtx; - pthread_cond_t finished; + TdThreadMutex mtx; + TdThreadCond finished; } IndexCache; #define CACHE_VERSION(cache) atomic_load_32(&cache->version) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 5da0dc537bfbefbe79a430d0220cef3149eb7f7b..cf5c3f306b5e56e15221ede6436ce0340be80ccd 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -260,7 +260,7 @@ typedef struct Fst { FstMeta* meta; FstSlice* data; // FstNode* root; // - pthread_mutex_t mtx; + TdThreadMutex mtx; } Fst; // refactor simple function diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index d25fed6816b456f31d3f141ceae2ad9281de4f4f..61dd9523811d0eb5a0fa330f72fc4a350cf2e2c5 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -45,7 +45,7 @@ typedef struct SIdxColInfo { int cVersion; } SIdxColInfo; -static pthread_once_t isInit = PTHREAD_ONCE_INIT; +static TdThreadOnce isInit = PTHREAD_ONCE_INIT; // static void indexInit(); static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); @@ -61,8 +61,8 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, Iterat // int32_t indexSerialKey(ICacheKey* key, char* buf); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { - pthread_once(&isInit, indexInit); - SIndex* sIdx = calloc(1, sizeof(SIndex)); + taosThreadOnce(&isInit, indexInit); + SIndex* sIdx = taosMemoryCalloc(1, sizeof(SIndex)); if (sIdx == NULL) { return -1; } @@ -82,7 +82,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; sIdx->path = tstrdup(path); - pthread_mutex_init(&sIdx->mtx, NULL); + taosThreadMutexInit(&sIdx->mtx, NULL); *index = sIdx; return 0; #endif @@ -112,11 +112,11 @@ void indexClose(SIndex* sIdx) { iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); - pthread_mutex_destroy(&sIdx->mtx); + taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); #endif - free(sIdx->path); - free(sIdx); + taosMemoryFree(sIdx->path); + taosMemoryFree(sIdx); return; } @@ -140,7 +140,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { #ifdef USE_INVERTED_INDEX // TODO(yihao): reduce the lock range - pthread_mutex_lock(&index->mtx); + taosThreadMutexLock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -154,7 +154,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } - pthread_mutex_unlock(&index->mtx); + taosThreadMutexUnlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -179,16 +179,16 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result EIndexOperatorType opera = multiQuerys->opera; int nQuery = taosArrayGetSize(multiQuerys->query); - char** fields = malloc(sizeof(char*) * nQuery); - char** keys = malloc(sizeof(char*) * nQuery); - int* types = malloc(sizeof(int) * nQuery); + char** fields = taosMemoryMalloc(sizeof(char*) * nQuery); + char** keys = taosMemoryMalloc(sizeof(char*) * nQuery); + int* types = taosMemoryMalloc(sizeof(int) * nQuery); for (int i = 0; i < nQuery; i++) { SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); SIndexTerm* term = p->field_value; - fields[i] = calloc(1, term->nKey + 1); - keys[i] = calloc(1, term->nVal + 1); + fields[i] = taosMemoryCalloc(1, term->nKey + 1); + keys[i] = taosMemoryCalloc(1, term->nVal + 1); memcpy(fields[i], term->key, term->nKey); memcpy(keys[i], term->val, term->nVal); @@ -203,12 +203,12 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result } for (int i = 0; i < nQuery; i++) { - free(fields[i]); - free(keys[i]); + taosMemoryFree(fields[i]); + taosMemoryFree(keys[i]); } - free(fields); - free(keys); - free(types); + taosMemoryFree(fields); + taosMemoryFree(keys); + taosMemoryFree(types); #endif #ifdef USE_INVERTED_INDEX @@ -258,7 +258,7 @@ void indexOptsDestroy(SIndexOpts* opts) { * */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); + SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); if (p == NULL) { return NULL; } @@ -272,7 +272,7 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { indexTermDestroy(p->term); } taosArrayDestroy(pQuery->query); - free(pQuery); + taosMemoryFree(pQuery); }; int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) { SIndexTermQuery q = {.qType = qType, .term = term}; @@ -282,7 +282,7 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { - SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); + SIndexTerm* t = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (t == NULL) { return NULL; } @@ -291,19 +291,19 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy t->operType = oper; t->colType = colType; - t->colName = (char*)calloc(1, nColName + 1); + t->colName = (char*)taosMemoryCalloc(1, nColName + 1); memcpy(t->colName, colName, nColName); t->nColName = nColName; - t->colVal = (char*)calloc(1, nColVal + 1); + t->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); memcpy(t->colVal, colVal, nColVal); t->nColVal = nColVal; return t; } void indexTermDestroy(SIndexTerm* p) { - free(p->colName); - free(p->colVal); - free(p); + taosMemoryFree(p->colName); + taosMemoryFree(p->colVal); + taosMemoryFree(p); } SIndexMultiTerm* indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm*)); } @@ -333,10 +333,10 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; int32_t sz = indexSerialCacheKey(&key, buf); - pthread_mutex_lock(&sIdx->mtx); + taosThreadMutexLock(&sIdx->mtx); IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); cache = (pCache == NULL) ? NULL : *pCache; - pthread_mutex_unlock(&sIdx->mtx); + taosThreadMutexUnlock(&sIdx->mtx); *result = taosArrayInit(4, sizeof(uint64_t)); // TODO: iterator mem and tidex @@ -536,7 +536,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { taosArrayClear(value->val); } } - free(value->colVal); + taosMemoryFree(value->colVal); value->colVal = NULL; } static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { @@ -564,15 +564,15 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { TFileHeader* header = &reader->header; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; - pthread_mutex_lock(&sIdx->mtx); + taosThreadMutexLock(&sIdx->mtx); IndexTFile* ifile = (IndexTFile*)sIdx->tindex; tfileCachePut(ifile->cache, &key, reader); - pthread_mutex_unlock(&sIdx->mtx); + taosThreadMutexUnlock(&sIdx->mtx); return ret; END: if (tw != NULL) { writerCtxDestroy(tw->ctx, true); - free(tw); + taosMemoryFree(tw); } return -1; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 34f009dd7e92167e2e6a1515c0fcdc725b5aa14d..bf907726bc974fbf7be7d2b074ba1c1320f38847 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -40,7 +40,7 @@ static bool indexCacheIteratorNext(Iterate* itera); static IterateValue* indexCacheIteratorGetValue(Iterate* iter); IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { - IndexCache* cache = calloc(1, sizeof(IndexCache)); + IndexCache* cache = taosMemoryCalloc(1, sizeof(IndexCache)); if (cache == NULL) { indexError("failed to create index cache"); return NULL; @@ -54,8 +54,8 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in cache->suid = suid; cache->occupiedMem = 0; - pthread_mutex_init(&cache->mtx, NULL); - pthread_cond_init(&cache->finished, NULL); + taosThreadMutexInit(&cache->mtx, NULL); + taosThreadCondInit(&cache->finished, NULL); indexCacheRef(cache); return cache; @@ -63,10 +63,10 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in void indexCacheDebug(IndexCache* cache) { MemTable* tbl = NULL; - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->mem; indexMemRef(tbl); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); { SSkipList* slt = tbl->mem; @@ -85,10 +85,10 @@ void indexCacheDebug(IndexCache* cache) { } { - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->imm; indexMemRef(tbl); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); if (tbl != NULL) { SSkipList* slt = tbl->mem; SSkipListIterator* iter = tSkipListCreateIter(slt); @@ -113,8 +113,8 @@ void indexCacheDestroySkiplist(SSkipList* slt) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); if (ct != NULL) { - free(ct->colVal); - free(ct); + taosMemoryFree(ct->colVal); + taosMemoryFree(ct); } } tSkipListDestroyIter(iter); @@ -126,13 +126,13 @@ void indexCacheDestroyImm(IndexCache* cache) { } MemTable* tbl = NULL; - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread - pthread_cond_broadcast(&cache->finished); + taosThreadCondBroadcast(&cache->finished); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); indexMemUnRef(tbl); indexMemUnRef(tbl); @@ -144,21 +144,21 @@ void indexCacheDestroy(void* cache) { } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); - free(pCache->colName); + taosMemoryFree(pCache->colName); - pthread_mutex_destroy(&pCache->mtx); - pthread_cond_destroy(&pCache->finished); + taosThreadMutexDestroy(&pCache->mtx); + taosThreadCondDestroy(&pCache->finished); - free(pCache); + taosMemoryFree(pCache); } Iterate* indexCacheIteratorCreate(IndexCache* cache) { - Iterate* iiter = calloc(1, sizeof(Iterate)); + Iterate* iiter = taosMemoryCalloc(1, sizeof(Iterate)); if (iiter == NULL) { return NULL; } - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); indexMemRef(cache->imm); @@ -169,7 +169,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { iiter->next = indexCacheIteratorNext; iiter->getValue = indexCacheIteratorGetValue; - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); return iiter; } @@ -179,7 +179,7 @@ void indexCacheIteratorDestroy(Iterate* iter) { } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); - free(iter); + taosMemoryFree(iter); } int indexCacheSchedToMerge(IndexCache* pCache) { @@ -200,7 +200,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - pthread_cond_wait(&cache->finished, &cache->mtx); + taosThreadCondWait(&cache->finished, &cache->mtx); } else { indexCacheRef(cache); cache->imm = cache->mem; @@ -221,7 +221,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { IndexCache* pCache = cache; indexCacheRef(pCache); // encode data - CacheTerm* ct = calloc(1, sizeof(CacheTerm)); + CacheTerm* ct = taosMemoryCalloc(1, sizeof(CacheTerm)); if (cache == NULL) { return -1; } @@ -230,7 +230,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (hasJson) { ct->colVal = indexPackJsonData(term); } else { - ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); + ct->colVal = (char*)taosMemoryCalloc(1, sizeof(char) * (term->nColVal + 1)); memcpy(ct->colVal, term->colVal, term->nColVal); } ct->version = atomic_add_fetch_32(&pCache->version, 1); @@ -240,7 +240,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { // ugly code, refactor later int64_t estimate = sizeof(ct) + strlen(ct->colVal); - pthread_mutex_lock(&pCache->mtx); + taosThreadMutexLock(&pCache->mtx); pCache->occupiedMem += estimate; indexCacheMakeRoomForWrite(pCache); MemTable* tbl = pCache->mem; @@ -248,7 +248,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { tSkipListPut(tbl->mem, (char*)ct); indexMemUnRef(tbl); - pthread_mutex_unlock(&pCache->mtx); + taosThreadMutexUnlock(&pCache->mtx); indexCacheUnRef(pCache); return 0; @@ -299,12 +299,12 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; - pthread_mutex_lock(&pCache->mtx); + taosThreadMutexLock(&pCache->mtx); mem = pCache->mem; imm = pCache->imm; indexMemRef(mem); indexMemRef(imm); - pthread_mutex_unlock(&pCache->mtx); + taosThreadMutexUnlock(&pCache->mtx); SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; @@ -323,7 +323,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result } if (hasJson) { - tfree(p); + taosMemoryFreeClear(p); } indexMemUnRef(mem); @@ -365,7 +365,7 @@ void indexMemUnRef(MemTable* tbl) { if (ref == 0) { SSkipList* slt = tbl->mem; indexCacheDestroySkiplist(slt); - free(tbl); + taosMemoryFree(tbl); } } @@ -373,8 +373,8 @@ static void indexCacheTermDestroy(CacheTerm* ct) { if (ct == NULL) { return; } - free(ct->colVal); - free(ct); + taosMemoryFree(ct->colVal); + taosMemoryFree(ct); } static char* indexCacheTermGet(const void* pData) { CacheTerm* p = (CacheTerm*)pData; @@ -394,7 +394,7 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) { static MemTable* indexInternalCacheCreate(int8_t type) { type = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : type; - MemTable* tbl = calloc(1, sizeof(MemTable)); + MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); indexMemRef(tbl); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, indexCacheTermCompare, SL_ALLOW_DUP_KEY, diff --git a/source/libs/index/src/index_comm.c b/source/libs/index/src/index_comm.c index 4f3cbaa4da3e295ceb3a743b22543faf81388752..4dea5fa0118cde2ee7c4a649ab6b57056cbace67 100644 --- a/source/libs/index/src/index_comm.c +++ b/source/libs/index/src/index_comm.c @@ -27,7 +27,7 @@ char* indexPackJsonData(SIndexTerm* itm) { uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; - char* buf = (char*)calloc(1, sz); + char* buf = (char*)taosMemoryCalloc(1, sz); char* p = buf; memcpy(p, itm->colName, itm->nColName); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 18cde151d2fefe1efde339d8c382512b885f3efb..09f382bbdc818de55a4bc77650907b1e08c68e5e 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -30,7 +30,7 @@ static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, Compi } FstUnFinishedNodes* fstUnFinishedNodesCreate() { - FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); + FstUnFinishedNodes* nodes = taosMemoryMalloc(sizeof(FstUnFinishedNodes)); if (nodes == NULL) { return NULL; } @@ -42,7 +42,7 @@ FstUnFinishedNodes* fstUnFinishedNodesCreate() { static void unFinishedNodeDestroyElem(void* elem) { FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; fstBuilderNodeDestroy(b->node); - free(b->last); + taosMemoryFree(b->last); b->last = NULL; } void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { @@ -51,11 +51,11 @@ void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { } taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); - free(nodes); + taosMemoryFree(nodes); } void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { - FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); node->isFinal = isFinal; node->finalOutput = 0; node->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -74,7 +74,7 @@ FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); - // free(un->last); // TODO add func FstLastTransitionFree() + // taosMemoryFree(un->last); // TODO add func FstLastTransitionFree() // un->last = NULL; return un->node; } @@ -103,7 +103,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); - // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); // trn->inp = s->data[s->start]; // trn->out = out; int32_t len = 0; @@ -111,12 +111,12 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output un->last = fstLastTransitionCreate(data[0], out); for (uint64_t i = 1; i < len; i++) { - FstBuilderNode* n = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* n = taosMemoryMalloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; n->trans = taosArrayInit(16, sizeof(FstTransition)); - // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); // trn->inp = s->data[i]; // trn->out = out; FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); @@ -296,7 +296,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil // at that index. (Except when there are 256 transitions.) Namely, // any value greater than or equal to the number of transitions in // this node indicates an absent transition. - uint8_t* index = (uint8_t*)malloc(sizeof(uint8_t) * 256); + uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256); memset(index, 255, sizeof(uint8_t) * 256); /// for (uint8_t i = 0; i < 256; i++) { // index[i] = 255; @@ -307,7 +307,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil // fstPackDeltaIn(w, addr, t->addr, tSize); } fstCountingWriterWrite(w, (char*)index, 256); - free(index); + taosMemoryFree(index); } fstCountingWriterWrite(w, (char*)&packSizes, 1); bool null = false; @@ -578,7 +578,7 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { // fst node function FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { - FstNode* n = (FstNode*)malloc(sizeof(FstNode)); + FstNode* n = (FstNode*)taosMemoryMalloc(sizeof(FstNode)); if (n == NULL) { return NULL; } @@ -643,10 +643,10 @@ static const char* fstNodeState(FstNode* node) { void fstNodeDestroy(FstNode* node) { fstSliceDestroy(&node->data); - free(node); + taosMemoryFree(node); } FstTransitions* fstNodeTransitions(FstNode* node) { - FstTransitions* t = malloc(sizeof(FstTransitions)); + FstTransitions* t = taosMemoryMalloc(sizeof(FstTransitions)); if (NULL == t) { return NULL; } @@ -755,7 +755,7 @@ bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, Compiled } FstBuilder* fstBuilderCreate(void* w, FstType ty) { - FstBuilder* b = malloc(sizeof(FstBuilder)); + FstBuilder* b = taosMemoryMalloc(sizeof(FstBuilder)); if (NULL == b) { return b; } @@ -788,7 +788,7 @@ void fstBuilderDestroy(FstBuilder* b) { fstUnFinishedNodesDestroy(b->unfinished); fstRegistryDestroy(b->registry); fstSliceDestroy(&b->last); - free(b); + taosMemoryFree(b); } bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { @@ -928,7 +928,7 @@ FstSlice fstNodeAsSlice(FstNode* node) { } FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { - FstLastTransition* trn = malloc(sizeof(FstLastTransition)); + FstLastTransition* trn = taosMemoryMalloc(sizeof(FstLastTransition)); if (trn == NULL) { return NULL; } @@ -938,7 +938,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { return trn; } -void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } +void fstLastTransitionDestroy(FstLastTransition* trn) { taosMemoryFree(trn); } void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; @@ -1003,12 +1003,12 @@ Fst* fstCreate(FstSlice* slice) { len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); // TODO(validate root addr) - Fst* fst = (Fst*)calloc(1, sizeof(Fst)); + Fst* fst = (Fst*)taosMemoryCalloc(1, sizeof(Fst)); if (fst == NULL) { return NULL; } - fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); + fst->meta = (FstMeta*)taosMemoryMalloc(sizeof(FstMeta)); if (NULL == fst->meta) { goto FST_CREAT_FAILED; } @@ -1019,32 +1019,32 @@ Fst* fstCreate(FstSlice* slice) { fst->meta->len = fstLen; fst->meta->checkSum = checkSum; - FstSlice* s = calloc(1, sizeof(FstSlice)); + FstSlice* s = taosMemoryCalloc(1, sizeof(FstSlice)); *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); fst->data = s; - pthread_mutex_init(&fst->mtx, NULL); + taosThreadMutexInit(&fst->mtx, NULL); return fst; FST_CREAT_FAILED: - free(fst->meta); - free(fst); + taosMemoryFree(fst->meta); + taosMemoryFree(fst); return NULL; } void fstDestroy(Fst* fst) { if (fst) { - free(fst->meta); + taosMemoryFree(fst->meta); fstSliceDestroy(fst->data); - free(fst->data); - pthread_mutex_destroy(&fst->mtx); + taosMemoryFree(fst->data); + taosThreadMutexDestroy(&fst->mtx); } - free(fst); + taosMemoryFree(fst); } bool fstGet(Fst* fst, FstSlice* b, Output* out) { // dec lock range - // pthread_mutex_lock(&fst->mtx); + // taosThreadMutexLock(&fst->mtx); FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; @@ -1057,7 +1057,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { uint8_t inp = data[i]; Output res = 0; if (false == fstNodeFindInput(root, inp, &res)) { - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); return false; } @@ -1068,7 +1068,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); @@ -1080,7 +1080,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { } taosArrayDestroy(nodes); // fst->root = NULL; - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); *out = tOut; return true; } @@ -1136,7 +1136,7 @@ bool fstVerify(Fst* fst) { // data bound function FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { - FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); + FstBoundWithData* b = taosMemoryCalloc(1, sizeof(FstBoundWithData)); if (b == NULL) { return NULL; } @@ -1171,11 +1171,11 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } -void fstBoundDestroy(FstBoundWithData* bound) { free(bound); } +void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); } StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { - StreamWithState* sws = calloc(1, sizeof(StreamWithState)); + StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState)); if (sws == NULL) { return NULL; } @@ -1201,7 +1201,7 @@ void streamWithStateDestroy(StreamWithState* sws) { taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); - free(sws); + taosMemoryFree(sws); } bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { @@ -1346,7 +1346,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb taosArrayPush(sws->stack, &s2); size_t isz = taosArrayGetSize(sws->inp); - uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); + uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } @@ -1354,19 +1354,19 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); taosArrayDestroy(nodes); return result; } - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { @@ -1378,7 +1378,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb } StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { - StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); + StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult)); if (result == NULL) { return NULL; } @@ -1395,7 +1395,7 @@ void swsResultDestroy(StreamWithStateResult* result) { fstSliceDestroy(&result->data); startWithStateValueDestroy(result->state); - free(result); + taosMemoryFree(result); } void streamStateDestroy(void* s) { @@ -1407,7 +1407,7 @@ void streamStateDestroy(void* s) { } FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { - FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); + FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder)); if (NULL == b) { return NULL; } @@ -1421,9 +1421,9 @@ FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { void fstStreamBuilderDestroy(FstStreamBuilder* b) { fstSliceDestroy(&b->min->data); fstSliceDestroy(&b->max->data); - tfree(b->min); - tfree(b->max); - free(b); + taosMemoryFreeClear(b->min); + taosMemoryFreeClear(b->max); + taosMemoryFree(b); } FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { if (b == NULL) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index ed1ad7a374286c8251dcb719be9a5510c6eec844..668a527d4a56e675f0b4ddc1a1393c74e20f7732 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,7 +16,7 @@ #include "index_fst_automation.h" StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { - StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue)); + StartWithStateValue* sv = taosMemoryCalloc(1, sizeof(StartWithStateValue)); if (sv == NULL) { return NULL; } @@ -27,7 +27,7 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp sv->val = *(int*)val; } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); - sv->ptr = (char*)calloc(1, len + 1); + sv->ptr = (char*)taosMemoryCalloc(1, len + 1); memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, @@ -44,14 +44,14 @@ void startWithStateValueDestroy(void* val) { if (sv->type == FST_INT) { // } else if (sv->type == FST_CHAR) { - free(sv->ptr); + taosMemoryFree(sv->ptr); } else if (sv->type == FST_ARRAY) { taosArrayDestroy(sv->arr); } - free(sv); + taosMemoryFree(sv); } StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { - StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); + StartWithStateValue* nsv = taosMemoryCalloc(1, sizeof(StartWithStateValue)); if (nsv == NULL) { return NULL; } @@ -62,7 +62,7 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { nsv->val = sv->val; } else if (nsv->type == FST_CHAR) { size_t len = strlen(sv->ptr); - nsv->ptr = (char*)calloc(1, len + 1); + nsv->ptr = (char*)taosMemoryCalloc(1, len + 1); memcpy(nsv->ptr, sv->ptr, len); } else if (nsv->type == FST_ARRAY) { // @@ -137,7 +137,7 @@ AutomationFunc automFuncs[] = { }; AutomationCtx* automCtxCreate(void* data, AutomationType atype) { - AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); + AutomationCtx* ctx = taosMemoryCalloc(1, sizeof(AutomationCtx)); if (ctx == NULL) { return NULL; } @@ -158,7 +158,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (data != NULL) { char* src = (char*)data; size_t len = strlen(src); - dst = (char*)calloc(1, len * sizeof(char) + 1); + dst = (char*)taosMemoryCalloc(1, len * sizeof(char) + 1); memcpy(dst, src, len); } @@ -169,6 +169,6 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { } void automCtxDestroy(AutomationCtx* ctx) { startWithStateValueDestroy(ctx->stdata); - free(ctx->data); - free(ctx); + taosMemoryFree(ctx->data); + taosMemoryFree(ctx); } diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 8cb2ff92461382fc2549f792c2cd326650946f04..6a161ba7e954085f753e0137b41fb98e2dc2b230 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -81,7 +81,7 @@ static int writeCtxDoFlush(WriterCtx* ctx) { } WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) { - WriterCtx* ctx = calloc(1, sizeof(WriterCtx)); + WriterCtx* ctx = taosMemoryCalloc(1, sizeof(WriterCtx)); if (ctx == NULL) { return NULL; } ctx->type = type; @@ -112,7 +112,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int goto END; } } else if (ctx->type == TMemory) { - ctx->mem.buf = calloc(1, sizeof(char) * capacity); + ctx->mem.buf = taosMemoryCalloc(1, sizeof(char) * capacity); ctx->mem.capa = capacity; } ctx->write = writeCtxDoWrite; @@ -126,13 +126,13 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int return ctx; END: - if (ctx->type == TMemory) { free(ctx->mem.buf); } - free(ctx); + if (ctx->type == TMemory) { taosMemoryFree(ctx->mem.buf); } + taosMemoryFree(ctx); return NULL; } void writerCtxDestroy(WriterCtx* ctx, bool remove) { if (ctx->type == TMemory) { - free(ctx->mem.buf); + taosMemoryFree(ctx->mem.buf); } else { ctx->flush(ctx); taosCloseFile(&ctx->file.pFile); @@ -150,11 +150,11 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { } if (remove) { unlink(ctx->file.buf); } } - free(ctx); + taosMemoryFree(ctx); } FstCountingWriter* fstCountingWriterCreate(void* wrt) { - FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter)); + FstCountingWriter* cw = taosMemoryCalloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } cw->wrt = wrt; @@ -165,7 +165,7 @@ void fstCountingWriterDestroy(FstCountingWriter* cw) { // free wrt object: close fd or free mem fstCountingWriterFlush(cw); // writerCtxDestroy((WriterCtx *)(cw->wrt)); - free(cw); + taosMemoryFree(cw); } int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) { @@ -203,13 +203,13 @@ int fstCountingWriterFlush(FstCountingWriter* write) { void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) { assert(1 <= nBytes && nBytes <= 8); - uint8_t* buf = calloc(8, sizeof(uint8_t)); + uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t)); for (uint8_t i = 0; i < nBytes; i++) { buf[i] = (uint8_t)n; n = n >> 8; } fstCountingWriterWrite(writer, buf, nBytes); - free(buf); + taosMemoryFree(buf); return; } diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index 5cdf6adb31a421e38fb22d30fe34599f1489924c..a0d59150d73231cddd2781ad6627e5d29c5e0422 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -15,7 +15,7 @@ #include "index_fst_node.h" FstBuilderNode* fstBuilderNodeDefault() { - FstBuilderNode* bn = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* bn = taosMemoryMalloc(sizeof(FstBuilderNode)); bn->isFinal = false; bn->finalOutput = 0; bn->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -25,7 +25,7 @@ void fstBuilderNodeDestroy(FstBuilderNode* node) { if (node == NULL) { return; } taosArrayDestroy(node->trans); - free(node); + taosMemoryFree(node); } bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { @@ -45,7 +45,7 @@ bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { return true; } FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) { - FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); if (node == NULL) { return NULL; } // diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index b28b518fc156b5d7585a2e071b603a186bcd54e7..7b6b8df25f5b7ad85d3d181a3a9ba6b9d8c9fbe8 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -65,13 +65,13 @@ static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { } FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { - FstRegistry* registry = malloc(sizeof(FstRegistry)); + FstRegistry* registry = taosMemoryMalloc(sizeof(FstRegistry)); if (registry == NULL) { return NULL; } uint64_t nCells = tableSize * mruSize; SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell)); if (NULL == tb) { - free(registry); + taosMemoryFree(registry); return NULL; } @@ -96,7 +96,7 @@ void fstRegistryDestroy(FstRegistry* registry) { fstBuilderNodeDestroy(cell->node); } taosArrayDestroy(tb); - free(registry); + taosMemoryFree(registry); } FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) { @@ -105,7 +105,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo uint64_t start = registry->mruSize * bucket; uint64_t end = start + registry->mruSize; - FstRegistryEntry* entry = malloc(sizeof(FstRegistryEntry)); + FstRegistryEntry* entry = taosMemoryMalloc(sizeof(FstRegistryEntry)); if (end - start == 1) { FstRegistryCell* cell = taosArrayGet(registry->table, start); // cell->isNode && @@ -166,5 +166,5 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo return entry; } void fstRegistryEntryDestroy(FstRegistryEntry* entry) { - free(entry); + taosMemoryFree(entry); } diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index f08a48c34ed6040b21b98f62e0514f0d0664c2bb..f9581f7202dc127d62f09ca5bf8b67f3ff4341b2 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -78,10 +78,10 @@ CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) { // FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { - FstString* str = (FstString*)malloc(sizeof(FstString)); + FstString* str = (FstString*)taosMemoryMalloc(sizeof(FstString)); str->ref = 1; str->len = len; - str->data = malloc(len * sizeof(uint8_t)); + str->data = taosMemoryMalloc(len * sizeof(uint8_t)); memcpy(str->data, data, len); FstSlice s = {.str = str, .start = 0, .end = len - 1}; @@ -101,10 +101,10 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { uint8_t* data = fstSliceData(s, &slen); assert(tlen <= slen); - uint8_t* buf = malloc(sizeof(uint8_t) * tlen); + uint8_t* buf = taosMemoryMalloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); - FstString* str = malloc(sizeof(FstString)); + FstString* str = taosMemoryMalloc(sizeof(FstString)); str->data = buf; str->len = tlen; str->ref = 1; @@ -128,8 +128,8 @@ void fstSliceDestroy(FstSlice* s) { FstString* str = s->str; str->ref--; if (str->ref == 0) { - free(str->data); - free(str); + taosMemoryFree(str->data); + taosMemoryFree(str); s->str = NULL; } } @@ -161,7 +161,7 @@ int fstSliceCompare(FstSlice* a, FstSlice* b) { } // FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { -// FstStack *s = calloc(1, sizeof(FstStack)); +// FstStack *s = taosMemoryCalloc(1, sizeof(FstStack)); // if (s == NULL) { return NULL; } // s-> // s->freeFn diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 780b7160fca098a851071871d5e79b3e3768a57d..53813e13e6920cacd435ef9ebc65525a2c80760d 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -59,7 +59,7 @@ static void tfileGenFileName(char* filename, uint64_t suid, const char* col, static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version); TFileCache* tfileCacheCreate(const char* path) { - TFileCache* tcache = calloc(1, sizeof(TFileCache)); + TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); if (tcache == NULL) { return NULL; } @@ -113,7 +113,7 @@ void tfileCacheDestroy(TFileCache* tcache) { reader = taosHashIterate(tcache->tableCache, reader); } taosHashCleanup(tcache->tableCache); - free(tcache); + taosMemoryFree(tcache); } TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { @@ -145,7 +145,7 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { return; } TFileReader* tfileReaderCreate(WriterCtx* ctx) { - TFileReader* reader = calloc(1, sizeof(TFileReader)); + TFileReader* reader = taosMemoryCalloc(1, sizeof(TFileReader)); if (reader == NULL) { return NULL; } @@ -181,7 +181,7 @@ void tfileReaderDestroy(TFileReader* reader) { // T_REF_INC(reader); fstDestroy(reader->fst); writerCtxDestroy(reader->ctx, reader->remove); - free(reader); + taosMemoryFree(reader); } int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { @@ -218,7 +218,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResul } fstSliceDestroy(&key); if (hasJson) { - free(p); + taosMemoryFree(p); } } else if (qtype == QUERY_PREFIX) { // handle later @@ -269,7 +269,7 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c return reader; } TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { - TFileWriter* tw = calloc(1, sizeof(TFileWriter)); + TFileWriter* tw = taosMemoryCalloc(1, sizeof(TFileWriter)); if (tw == NULL) { indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid); return NULL; @@ -296,7 +296,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { } int32_t bufLimit = 64 * 4096, offset = 0; - // char* buf = calloc(1, sizeof(char) * bufLimit); + // char* buf = taosMemoryCalloc(1, sizeof(char) * bufLimit); // char* p = buf; int32_t sz = taosArrayGetSize((SArray*)data); int32_t fstOffset = tw->offset; @@ -318,13 +318,13 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { // check buf has enough space or not int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); - char* buf = calloc(1, ttsz * sizeof(char)); + char* buf = taosMemoryCalloc(1, ttsz * sizeof(char)); char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); tw->ctx->write(tw->ctx, buf, ttsz); v->offset = tw->offset; tw->offset += ttsz; - free(buf); + taosMemoryFree(buf); } tw->fb = fstBuilderCreate(tw->ctx, 0); @@ -359,14 +359,14 @@ void tfileWriterClose(TFileWriter* tw) { return; } writerCtxDestroy(tw->ctx, false); - free(tw); + taosMemoryFree(tw); } void tfileWriterDestroy(TFileWriter* tw) { if (tw == NULL) { return; } writerCtxDestroy(tw->ctx, false); - free(tw); + taosMemoryFree(tw); } IndexTFile* indexTFileCreate(const char* path) { @@ -375,7 +375,7 @@ IndexTFile* indexTFileCreate(const char* path) { return NULL; } - IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); + IndexTFile* tfile = taosMemoryCalloc(1, sizeof(IndexTFile)); if (tfile == NULL) { tfileCacheDestroy(cache); return NULL; @@ -389,7 +389,7 @@ void indexTFileDestroy(IndexTFile* tfile) { return; } tfileCacheDestroy(tfile->cache); - free(tfile); + taosMemoryFree(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTempResult* result) { @@ -433,7 +433,7 @@ static bool tfileIteratorNext(Iterate* iiter) { int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); - colVal = calloc(1, sz + 1); + colVal = taosMemoryCalloc(1, sz + 1); memcpy(colVal, ch, sz); offset = (uint64_t)(rt->out.out); @@ -453,7 +453,7 @@ static bool tfileIteratorNext(Iterate* iiter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { - TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); + TFileFstIter* tIter = taosMemoryCalloc(1, sizeof(TFileFstIter)); if (tIter == NULL) { return NULL; } @@ -470,10 +470,10 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { return NULL; } - Iterate* iter = calloc(1, sizeof(Iterate)); + Iterate* iter = taosMemoryCalloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); if (iter->iter == NULL) { - free(iter); + taosMemoryFree(iter); return NULL; } iter->next = tfileIteratorNext; @@ -494,9 +494,9 @@ void tfileIteratorDestroy(Iterate* iter) { streamWithStateDestroy(tIter->st); fstStreamBuilderDestroy(tIter->fb); automCtxDestroy(tIter->ctx); - free(tIter); + taosMemoryFree(tIter); - free(iter); + taosMemoryFree(iter); } TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { @@ -530,7 +530,7 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { } TFileValue* tfileValueCreate(char* val) { - TFileValue* tf = calloc(1, sizeof(TFileValue)); + TFileValue* tf = taosMemoryCalloc(1, sizeof(TFileValue)); if (tf == NULL) { return NULL; } @@ -547,8 +547,8 @@ int tfileValuePush(TFileValue* tf, uint64_t val) { } void tfileValueDestroy(TFileValue* tf) { taosArrayDestroy(tf->tableId); - free(tf->colVal); - free(tf); + taosMemoryFree(tf->colVal); + taosMemoryFree(tf); } static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { int sz = taosArrayGetSize(ids); @@ -636,7 +636,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { // current load fst into memory, refactor it later int fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber); - char* buf = calloc(1, fstSize); + char* buf = taosMemoryCalloc(1, fstSize); if (buf == NULL) { return -1; } @@ -651,7 +651,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { FstSlice st = fstSliceCreate((uint8_t*)buf, nread); reader->fst = fstCreate(&st); - free(buf); + taosMemoryFree(buf); fstSliceDestroy(&st); return reader->fst != NULL ? 0 : -1; @@ -744,7 +744,7 @@ static SArray* tfileGetFileList(const char* path) { } size_t len = strlen(path) + 1 + strlen(file) + 1; - char* buf = calloc(1, len); + char* buf = taosMemoryCalloc(1, len); sprintf(buf, "%s/%s", path, file); taosArrayPush(files, &buf); } @@ -761,7 +761,7 @@ static int tfileRmExpireFile(SArray* result) { } static void tfileDestroyFileName(void* elem) { char* p = *(char**)elem; - free(p); + taosMemoryFree(p); } static int tfileCompare(const void* a, const void* b) { const char* as = *(char**)a; diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c index dfe4e273a9e09255d2016b874e76ed108fc011c4..65c16ca65ba5b54de2f3492abe4314a6419c4e16 100644 --- a/source/libs/index/src/index_util.c +++ b/source/libs/index/src/index_util.c @@ -41,7 +41,7 @@ void iIntersection(SArray *inters, SArray *final) { if (sz <= 0) { return; } - MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(inters, i); mi[i].len = taosArrayGetSize(t); @@ -67,7 +67,7 @@ void iIntersection(SArray *inters, SArray *final) { taosArrayPush(final, &tgt); } } - tfree(mi); + taosMemoryFreeClear(mi); } void iUnion(SArray *inters, SArray *final) { int32_t sz = taosArrayGetSize(inters); @@ -79,7 +79,7 @@ void iUnion(SArray *inters, SArray *final) { return; } - MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(inters, i); mi[i].len = taosArrayGetSize(t); @@ -113,7 +113,7 @@ void iUnion(SArray *inters, SArray *final) { break; } } - tfree(mi); + taosMemoryFreeClear(mi); } void iExcept(SArray *total, SArray *except) { @@ -156,7 +156,7 @@ int verdataCompare(const void *a, const void *b) { } SIdxTempResult *sIdxTempResultCreate() { - SIdxTempResult *tr = calloc(1, sizeof(SIdxTempResult)); + SIdxTempResult *tr = taosMemoryCalloc(1, sizeof(SIdxTempResult)); tr->total = taosArrayInit(4, sizeof(uint64_t)); tr->added = taosArrayInit(4, sizeof(uint64_t)); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 410d5b702125df60c780fbcf5a90951dc9d24323..94923726ddb4667be7352786916135739bd4ed13 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -55,7 +55,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -63,7 +63,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFree(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { @@ -230,7 +230,7 @@ void checkFstLongTerm() { // for (int i = 0; i < result.size(); i++) { // assert(result[i] == i); // check result //} - // free(ctx); + // taosMemoryFree(ctx); // delete m; } void checkFstCheckIterator() { @@ -266,7 +266,7 @@ void checkFstCheckIterator() { // assert(result[i] == i); // check result } - free(ctx); + taosMemoryFree(ctx); delete m; } diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index 6e9c88bc7953c4d7d2b6e8b142198b606290dc4d..1bdc7fc9c9a0c4f2081c6f36038fee47c5fd3338 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -75,7 +75,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -83,7 +83,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFreeClear(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 5f339f286576f3fd5d535470ed9583a2b65363df..0e4eb060cf06e8414cd41b048ff44fdc7feab751 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -81,7 +81,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -89,7 +89,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFree(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { @@ -227,7 +227,7 @@ void checkFstPrefixSearch() { assert(result[i] == i); // check result } - free(ctx); + taosMemoryFree(ctx); delete m; } void validateFst() { @@ -446,9 +446,9 @@ class IndexTFileEnv : public ::testing::Test { }; static TFileValue* genTFileValue(const char* val) { - TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue)); + TFileValue* tv = (TFileValue*)taosMemoryCalloc(1, sizeof(TFileValue)); int32_t vlen = strlen(val) + 1; - tv->colVal = (char*)calloc(1, vlen); + tv->colVal = (char*)taosMemoryCalloc(1, vlen); memcpy(tv->colVal, val, vlen); tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -460,9 +460,9 @@ static TFileValue* genTFileValue(const char* val) { } static void destroyTFileValue(void* val) { TFileValue* tv = (TFileValue*)val; - free(tv->colVal); + taosMemoryFree(tv->colVal); taosArrayDestroy(tv->tableId); - free(tv); + taosMemoryFree(tv); } TEST_F(IndexTFileEnv, test_tfile_write) { TFileValue* v1 = genTFileValue("ab"); diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index 6ef901410bff1923ee425ffcaff6c44eb7d047d8..452c38f66b58d13425156d4986d71d6b54d5cc63 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -48,7 +48,7 @@ typedef struct SMonInfo { } SMonInfo; typedef struct { - pthread_mutex_t lock; + TdThreadMutex lock; SArray *logs; // array of SMonLogItem int32_t maxLogs; const char *server; diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 352b59e9316d755d51619e7499d73e53441536e3..b92c08d51cbc23b9a22cd8a0f56766cb2852b226 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -23,7 +23,7 @@ static SMonitor tsMonitor = {0}; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { - pthread_mutex_lock(&tsMonitor.lock); + taosThreadMutexLock(&tsMonitor.lock); int32_t size = taosArrayGetSize(tsMonitor.logs); if (size < tsMonitor.maxLogs) { SMonLogItem item = {.ts = ts, .level = level}; @@ -32,7 +32,7 @@ void monRecordLog(int64_t ts, ELogLevel level, const char *content) { tstrncpy(pItem->content, content, MON_LOG_LEN); } } - pthread_mutex_unlock(&tsMonitor.lock); + taosThreadMutexUnlock(&tsMonitor.lock); } int32_t monInit(const SMonCfg *pCfg) { @@ -48,7 +48,7 @@ int32_t monInit(const SMonCfg *pCfg) { tsMonitor.comp = pCfg->comp; tsLogFp = monRecordLog; tsMonitor.state.time = taosGetTimestampMs(); - pthread_mutex_init(&tsMonitor.lock, NULL); + taosThreadMutexInit(&tsMonitor.lock, NULL); return 0; } @@ -56,20 +56,20 @@ void monCleanup() { tsLogFp = NULL; taosArrayDestroy(tsMonitor.logs); tsMonitor.logs = NULL; - pthread_mutex_destroy(&tsMonitor.lock); + taosThreadMutexDestroy(&tsMonitor.lock); } SMonInfo *monCreateMonitorInfo() { - SMonInfo *pMonitor = calloc(1, sizeof(SMonInfo)); + SMonInfo *pMonitor = taosMemoryCalloc(1, sizeof(SMonInfo)); if (pMonitor == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pthread_mutex_lock(&tsMonitor.lock); + taosThreadMutexLock(&tsMonitor.lock); pMonitor->logs = taosArrayDup(tsMonitor.logs); taosArrayClear(tsMonitor.logs); - pthread_mutex_unlock(&tsMonitor.lock); + taosThreadMutexUnlock(&tsMonitor.lock); pMonitor->pJson = tjsonCreateObject(); if (pMonitor->pJson == NULL || pMonitor->logs == NULL) { @@ -88,7 +88,7 @@ void monCleanupMonitorInfo(SMonInfo *pMonitor) { tsMonitor.state.time = pMonitor->curTime; taosArrayDestroy(pMonitor->logs); tjsonDelete(pMonitor->pJson); - free(pMonitor); + taosMemoryFree(pMonitor); } void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { @@ -381,6 +381,6 @@ void monSendReport(SMonInfo *pMonitor) { if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.comp ? HTTP_GZIP : HTTP_FLAT; taosSendHttpReport(tsMonitor.server, tsMonitor.port, pCont, strlen(pCont), flag); - free(pCont); + taosMemoryFree(pCont); } } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index d66203eb40c6ea7ac6fe0de6f3df116d4c620a4f..a9660c8573c6e172078ca160c230c9800ae72841 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -31,6 +31,9 @@ #define COPY_CHAR_POINT_FIELD(fldname) \ do { \ + if (NULL == (pSrc)->fldname) { \ + break; \ + } \ (pDst)->fldname = strdup((pSrc)->fldname); \ } while (0) @@ -108,6 +111,10 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_POINT_FIELD(literal); COPY_SCALAR_FIELD(isDuration); + COPY_SCALAR_FIELD(translate); + if (!pSrc->translate) { + return (SNode*)pDst; + } switch (pSrc->node.resType.type) { case TSDB_DATA_TYPE_NULL: break; @@ -134,7 +141,12 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - COPY_CHAR_POINT_FIELD(datum.p); + pDst->datum.p = taosMemoryMalloc(pSrc->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + if (NULL == pDst->datum.p) { + nodesDestroyNode(pDst); + return NULL; + } + memcpy(pDst->datum.p, pSrc->datum.p, pSrc->node.resType.bytes + VARSTR_HEADER_SIZE + 1); break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: @@ -198,7 +210,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { static STableMeta* tableMetaClone(const STableMeta* pSrc) { int32_t len = TABLE_META_SIZE(pSrc); - STableMeta* pDst = malloc(len); + STableMeta* pDst = taosMemoryMalloc(len); if (NULL == pDst) { return NULL; } @@ -208,7 +220,7 @@ static STableMeta* tableMetaClone(const STableMeta* pSrc) { static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) { int32_t len = VGROUPS_INFO_SIZE(pSrc); - SVgroupsInfo* pDst = malloc(len); + SVgroupsInfo* pDst = taosMemoryMalloc(len); if (NULL == pDst) { return NULL; } @@ -225,6 +237,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(scanFlag); COPY_SCALAR_FIELD(scanRange); COPY_SCALAR_FIELD(tableName); + COPY_SCALAR_FIELD(showRewrite); return (SNode*)pDst; } @@ -260,11 +273,14 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD COPY_SCALAR_FIELD(interval); COPY_SCALAR_FIELD(offset); COPY_SCALAR_FIELD(sliding); + COPY_SCALAR_FIELD(intervalUnit); + COPY_SCALAR_FIELD(slidingUnit); CLONE_NODE_FIELD(pFill); + COPY_SCALAR_FIELD(sessionGap); return (SNode*)pDst; } -static SNode* logicSubplanCopy(const SSubLogicPlan* pSrc, SSubLogicPlan* pDst) { +static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) { CLONE_NODE_FIELD(pNode); COPY_SCALAR_FIELD(subplanType); return (SNode*)pDst; @@ -346,7 +362,7 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { case QUERY_NODE_LOGIC_PLAN_WINDOW: return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); case QUERY_NODE_LOGIC_SUBPLAN: - return logicSubplanCopy((const SSubLogicPlan*)pNode, (SSubLogicPlan*)pDst); + return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst); default: break; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c61a2d71fb21db461eaa38c17d0b1c9414fcb482..d7caf6e511ea2d21cfc43b6f67436ac1b2b1b2fd 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "cmdnodes.h" #include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" @@ -69,6 +70,14 @@ const char* nodesNodeName(ENodeType type) { return "SlotDesc"; case QUERY_NODE_COLUMN_DEF: return "ColumnDef"; + case QUERY_NODE_DOWNSTREAM_SOURCE: + return "DownstreamSource"; + case QUERY_NODE_DATABASE_OPTIONS: + return "DatabaseOptions"; + case QUERY_NODE_TABLE_OPTIONS: + return "TableOptions"; + case QUERY_NODE_INDEX_OPTIONS: + return "IndexOptions"; case QUERY_NODE_SET_OPERATOR: return "SetOperator"; case QUERY_NODE_SELECT_STMT: @@ -77,14 +86,76 @@ const char* nodesNodeName(ENodeType type) { return "VnodeModifStmt"; case QUERY_NODE_CREATE_DATABASE_STMT: return "CreateDatabaseStmt"; + case QUERY_NODE_DROP_DATABASE_STMT: + return "DropDatabaseStmt"; + case QUERY_NODE_ALTER_DATABASE_STMT: + return "AlterDatabaseStmt"; case QUERY_NODE_CREATE_TABLE_STMT: return "CreateTableStmt"; + case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: + return "CreateSubtableClause"; + case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + return "CreateMultiTableStmt"; + case QUERY_NODE_DROP_TABLE_CLAUSE: + return "DropTableClause"; + case QUERY_NODE_DROP_TABLE_STMT: + return "DropTableStmt"; + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return "DropSuperTableStmt"; + case QUERY_NODE_ALTER_TABLE_STMT: + return "AlterTableStmt"; + case QUERY_NODE_CREATE_USER_STMT: + return "CreateUserStmt"; + case QUERY_NODE_ALTER_USER_STMT: + return "AlterUserStmt"; + case QUERY_NODE_DROP_USER_STMT: + return "DropUserStmt"; case QUERY_NODE_USE_DATABASE_STMT: return "UseDatabaseStmt"; + case QUERY_NODE_CREATE_DNODE_STMT: + return "CreateDnodeStmt"; + case QUERY_NODE_DROP_DNODE_STMT: + return "DropDnodeStmt"; + case QUERY_NODE_ALTER_DNODE_STMT: + return "AlterDnodeStmt"; + case QUERY_NODE_CREATE_INDEX_STMT: + return "CreateIndexStmt"; + case QUERY_NODE_DROP_INDEX_STMT: + return "DropIndexStmt"; + case QUERY_NODE_CREATE_QNODE_STMT: + return "CreateQnodeStmt"; + case QUERY_NODE_DROP_QNODE_STMT: + return "DropQnodeStmt"; + case QUERY_NODE_CREATE_TOPIC_STMT: + return "CreateTopicStmt"; + case QUERY_NODE_DROP_TOPIC_STMT: + return "DropTopicStmt"; + case QUERY_NODE_ALTER_LOCAL_STMT: + return "AlterLocalStmt"; case QUERY_NODE_SHOW_DATABASES_STMT: return "ShowDatabaseStmt"; case QUERY_NODE_SHOW_TABLES_STMT: return "ShowTablesStmt"; + case QUERY_NODE_SHOW_STABLES_STMT: + return "ShowStablesStmt"; + case QUERY_NODE_SHOW_USERS_STMT: + return "ShowUsersStmt"; + case QUERY_NODE_SHOW_DNODES_STMT: + return "ShowDnodesStmt"; + case QUERY_NODE_SHOW_VGROUPS_STMT: + return "ShowVgroupsStmt"; + case QUERY_NODE_SHOW_MNODES_STMT: + return "ShowMnodesStmt"; + case QUERY_NODE_SHOW_MODULES_STMT: + return "ShowModulesStmt"; + case QUERY_NODE_SHOW_QNODES_STMT: + return "ShowQnodesStmt"; + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + return "ShowFunctionsStmt"; + case QUERY_NODE_SHOW_INDEXES_STMT: + return "ShowIndexesStmt"; + case QUERY_NODE_SHOW_STREAMS_STMT: + return "ShowStreamsStmt"; case QUERY_NODE_LOGIC_PLAN_SCAN: return "LogicScan"; case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -95,6 +166,10 @@ const char* nodesNodeName(ENodeType type) { return "LogicProject"; case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: return "LogicVnodeModif"; + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return "LogicExchange"; + case QUERY_NODE_LOGIC_PLAN_WINDOW: + return "LogicWindow"; case QUERY_NODE_LOGIC_SUBPLAN: return "LogicSubplan"; case QUERY_NODE_LOGIC_PLAN: @@ -107,6 +182,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiTableSeqScan"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: return "PhysiSreamScan"; + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + return "PhysiSystemTableScan"; case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return "PhysiProject"; case QUERY_NODE_PHYSICAL_PLAN_JOIN: @@ -119,6 +196,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return "PhysiInterval"; + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + return "PhysiSessionWindow"; case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return "PhysiDispatch"; case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -179,16 +258,118 @@ static int32_t jsonToNodeList(const SJson* pJson, const char* pName, SNodeList** return jsonToNodeListImpl(tjsonGetObjectItem(pJson, pName), pList); } -static const char* jkTableMetaUid = "TableMetaUid"; -static const char* jkTableMetaSuid = "TableMetaSuid"; +static const char* jkTableComInfoNumOfTags = "NumOfTags"; +static const char* jkTableComInfoPrecision = "Precision"; +static const char* jkTableComInfoNumOfColumns = "NumOfColumns"; +static const char* jkTableComInfoRowSize = "RowSize"; + +static int32_t tableComInfoToJson(const void* pObj, SJson* pJson) { + const STableComInfo* pNode = (const STableComInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfTags, pNode->numOfTags); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoPrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoRowSize, pNode->rowSize); + } + + return code; +} + +static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) { + STableComInfo* pNode = (STableComInfo*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize); + } + + return code; +} + +static const char* jkSchemaType = "Type"; +static const char* jkSchemaColId = "ColId"; +static const char* jkSchemaBytes = "bytes"; +static const char* jkSchemaName = "Name"; + +static int32_t schemaToJson(const void* pObj, SJson* pJson) { + const SSchema* pNode = (const SSchema*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSchemaType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSchemaColId, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSchemaBytes, pNode->bytes); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSchemaName, pNode->name); + } + + return code; +} + +static int32_t jsonToSchema(const SJson* pJson, void* pObj) { + SSchema* pNode = (SSchema*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkSchemaType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name); + } + + return code; +} + +static const char* jkTableMetaVgId = "VgId"; +static const char* jkTableMetaTableType = "TableType"; +static const char* jkTableMetaUid = "Uid"; +static const char* jkTableMetaSuid = "Suid"; +static const char* jkTableMetaSversion = "Sversion"; +static const char* jkTableMetaTversion = "Tversion"; +static const char* jkTableMetaComInfo = "ComInfo"; +static const char* jkTableMetaColSchemas = "ColSchemas"; static int32_t tableMetaToJson(const void* pObj, SJson* pJson) { const STableMeta* pNode = (const STableMeta*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid); + int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableMetaSuid, pNode->suid); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaSversion, pNode->sversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaTversion, pNode->tversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTableMetaComInfo, tableComInfoToJson, &pNode->tableInfo); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkTableMetaColSchemas, schemaToJson, pNode->schema, sizeof(SSchema), TABLE_TOTAL_COL_NUM(pNode)); + } return code; } @@ -196,9 +377,27 @@ static int32_t tableMetaToJson(const void* pObj, SJson* pJson) { static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { STableMeta* pNode = (STableMeta*)pObj; - int32_t code = tjsonGetUBigIntValue(pJson, jkTableMetaUid, &pNode->uid); + int32_t code = tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion); + } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUBigIntValue(pJson, jkTableMetaSuid, &pNode->suid); + code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkTableMetaColSchemas, jsonToSchema, pNode->schema, sizeof(SSchema)); } return code; @@ -222,7 +421,22 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { + SLogicNode* pNode = (SLogicNode*)pObj; + + int32_t code = jsonToNodeList(pJson, jkLogicPlanTargets, &pNode->pTargets); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkLogicPlanConditions, &pNode->pConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkLogicPlanChildren, &pNode->pChildren); + } + + return code; +} + static const char* jkScanLogicPlanScanCols = "ScanCols"; +static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize"; static const char* jkScanLogicPlanTableMeta = "TableMeta"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { @@ -232,6 +446,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta); } @@ -239,6 +456,24 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { + SScanLogicNode* pNode = (SScanLogicNode*)pObj; + + int32_t objSize = 0; + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanLogicPlanScanCols, &pNode->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize); + } + + return code; +} + static const char* jkProjectLogicPlanProjections = "Projections"; static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { @@ -252,6 +487,17 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { + SProjectLogicNode* pNode = (SProjectLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkProjectLogicPlanProjections, &pNode->pProjections); + } + + return code; +} + static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; @@ -460,6 +706,101 @@ static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiScanNode(pJson, pObj); } +static const char* jkEndPointFqdn = "Fqdn"; +static const char* jkEndPointPort = "Port"; + +static int32_t epToJson(const void* pObj, SJson* pJson) { + const SEp* pNode = (const SEp*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkEndPointFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkEndPointPort, pNode->port); + } + + return code; +} + +static int32_t jsonToEp(const SJson* pJson, void* pObj) { + SEp* pNode = (SEp*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkEndPointFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetSmallIntValue(pJson, jkEndPointPort, &pNode->port); + } + + return code; +} + +static const char* jkEpSetInUse = "InUse"; +static const char* jkEpSetNumOfEps = "NumOfEps"; +static const char* jkEpSetEps = "Eps"; + +static int32_t epSetToJson(const void* pObj, SJson* pJson) { + const SEpSet* pNode = (const SEpSet*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps); + } + + return code; +} + +static int32_t jsonToEpSet(const SJson* pJson, void* pObj) { + SEpSet* pNode = (SEpSet*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp)); + } + + return code; +} + +static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet"; +static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite"; +static const char* jkSysTableScanPhysiPlanAccountId = "AccountId"; + +static int32_t physiSysTableScanNodeToJson(const void* pObj, SJson* pJson) { + const SSystemTableScanPhysiNode* pNode = (const SSystemTableScanPhysiNode*)pObj; + + int32_t code = physiScanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSysTableScanPhysiPlanMnodeEpSet, epSetToJson, &pNode->mgmtEpSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSysTableScanPhysiPlanShowRewrite, pNode->showRewrite); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId); + } + + return code; +} + +static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) { + SSystemTableScanPhysiNode* pNode = (SSystemTableScanPhysiNode*)pObj; + + int32_t code = jsonToPhysiScanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkSysTableScanPhysiPlanMnodeEpSet, jsonToEpSet, &pNode->mgmtEpSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSysTableScanPhysiPlanShowRewrite, &pNode->showRewrite); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId); + } + + return code; +} + static const char* jkProjectPhysiPlanProjections = "Projections"; static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { @@ -593,8 +934,37 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { return code; } -static const char* jkIntervalPhysiPlanExprs = "Exprs"; -static const char* jkIntervalPhysiPlanFuncs = "Funcs"; +static const char* jkWindowPhysiPlanExprs = "Exprs"; +static const char* jkWindowPhysiPlanFuncs = "Funcs"; + +static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { + const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkWindowPhysiPlanExprs, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkWindowPhysiPlanFuncs, pNode->pFuncs); + } + + return code; +} + +static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { + SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkWindowPhysiPlanExprs, &pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkWindowPhysiPlanFuncs, &pNode->pFuncs); + } + + return code; +} + static const char* jkIntervalPhysiPlanInterval = "Interval"; static const char* jkIntervalPhysiPlanOffset = "Offset"; static const char* jkIntervalPhysiPlanSliding = "Sliding"; @@ -605,13 +975,7 @@ static const char* jkIntervalPhysiPlanFill = "Fill"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; - int32_t code = physicPlanNodeToJson(pObj, pJson); - if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkIntervalPhysiPlanExprs, pNode->pExprs); - } - if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkIntervalPhysiPlanFuncs, pNode->pFuncs); - } + int32_t code = physiWindowNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanInterval, pNode->interval); } @@ -637,13 +1001,7 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { SIntervalPhysiNode* pNode = (SIntervalPhysiNode*)pObj; - int32_t code = jsonToPhysicPlanNode(pJson, pObj); - if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkIntervalPhysiPlanExprs, &pNode->pExprs); - } - if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkIntervalPhysiPlanFuncs, &pNode->pFuncs); - } + int32_t code = jsonToPhysiWindowNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkIntervalPhysiPlanInterval, &pNode->interval); } @@ -666,6 +1024,30 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkSessionWindowPhysiPlanGap = "Gap"; + +static int32_t physiSessionWindowNodeToJson(const void* pObj, SJson* pJson) { + const SSessionWinodwPhysiNode* pNode = (const SSessionWinodwPhysiNode*)pObj; + + int32_t code = physiWindowNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSessionWindowPhysiPlanGap, pNode->gap); + } + + return code; +} + +static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) { + SSessionWinodwPhysiNode* pNode = (SSessionWinodwPhysiNode*)pObj; + + int32_t code = jsonToPhysiWindowNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap); + } + + return code; +} + static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc"; static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) { @@ -718,31 +1100,6 @@ static int32_t jsonToSubplanId(const SJson* pJson, void* pObj) { return code; } -static const char* jkEndPointFqdn = "Fqdn"; -static const char* jkEndPointPort = "Port"; - -static int32_t epToJson(const void* pObj, SJson* pJson) { - const SEp* pNode = (const SEp*)pObj; - - int32_t code = tjsonAddStringToObject(pJson, jkEndPointFqdn, pNode->fqdn); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkEndPointPort, pNode->port); - } - - return code; -} - -static int32_t jsonToEp(const SJson* pJson, void* pObj) { - SEp* pNode = (SEp*)pObj; - - int32_t code = tjsonGetStringValue(pJson, jkEndPointFqdn, pNode->fqdn); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetSmallIntValue(pJson, jkEndPointPort, &pNode->port); - } - - return code; -} - static const char* jkQueryNodeAddrId = "Id"; static const char* jkQueryNodeAddrInUse = "InUse"; static const char* jkQueryNodeAddrNumOfEps = "NumOfEps"; @@ -1074,7 +1431,7 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - code = tjsonAddStringToObject(pJson, jkValueDatum, pNode->datum.p); + code = tjsonAddStringToObject(pJson, jkValueDatum, varDataVal(pNode->datum.p)); break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: @@ -1136,9 +1493,16 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: - code = tjsonDupStringValue(pJson, jkValueDatum, &pNode->datum.p); + case TSDB_DATA_TYPE_VARBINARY: { + pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + if (NULL == pNode->datum.p) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + varDataSetLen(pNode->datum.p, pNode->node.resType.bytes); + code = tjsonGetStringValue(pJson, jkValueDatum, varDataVal(pNode->datum.p)); break; + } case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: @@ -1326,38 +1690,6 @@ static int32_t jsonToTableNode(const SJson* pJson, void* pObj) { return code; } -static const char* jkEpSetInUse = "InUse"; -static const char* jkEpSetNumOfEps = "NumOfEps"; -static const char* jkEpSetEps = "Eps"; - -static int32_t epSetToJson(const void* pObj, SJson* pJson) { - const SEpSet* pNode = (const SEpSet*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps); - } - - return code; -} - -static int32_t jsonToEpSet(const SJson* pJson, void* pObj) { - SEpSet* pNode = (SEpSet*)pObj; - - int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp)); - } - - return code; -} - static const char* jkVgroupInfoVgId = "VgId"; static const char* jkVgroupInfoHashBegin = "HashBegin"; static const char* jkVgroupInfoHashEnd = "HashEnd"; @@ -1489,6 +1821,45 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { return code; } +static const char* jkIntervalWindowInterval = "Interval"; +static const char* jkIntervalWindowOffset = "Offset"; +static const char* jkIntervalWindowSliding = "Sliding"; +static const char* jkIntervalWindowFill = "Fill"; + +static int32_t intervalWindowNodeToJson(const void* pObj, SJson* pJson) { + const SIntervalWindowNode* pNode = (const SIntervalWindowNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkIntervalWindowInterval, nodeToJson, pNode->pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowOffset, nodeToJson, pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowSliding, nodeToJson, pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowFill, nodeToJson, pNode->pFill); + } + + return code; +} + +static int32_t jsonToIntervalWindowNode(const SJson* pJson, void* pObj) { + SIntervalWindowNode* pNode = (SIntervalWindowNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkIntervalWindowInterval, &pNode->pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowOffset, &pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowSliding, &pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowFill, &pNode->pFill); + } + + return code; +} + static const char* jkNodeListDataType = "DataType"; static const char* jkNodeListNodeList = "NodeList"; @@ -1739,6 +2110,45 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) { return code; } +static const char* jkCreateTopicStmtTopicName = "TopicName"; +static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; +static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateTopicStmtQuery = "Query"; + +static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) { + const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery); + } + + return code; +} + +static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { + SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: @@ -1762,8 +2172,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_LIMIT: case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_SESSION_WINDOW: - case QUERY_NODE_INTERVAL_WINDOW: break; + case QUERY_NODE_INTERVAL_WINDOW: + return intervalWindowNodeToJson(pObj, pJson); case QUERY_NODE_NODE_LIST: return nodeListNodeToJson(pObj, pJson); case QUERY_NODE_FILL: @@ -1790,6 +2201,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_TABLES_STMT: break; + case QUERY_NODE_CREATE_TOPIC_STMT: + return createTopicStmtToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -1808,6 +2221,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiTableScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: return physiStreamScanNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + return physiSysTableScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return physiProjectNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_JOIN: @@ -1820,6 +2235,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { break; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + return physiSessionWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return physiDispatchNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -1829,7 +2246,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN: return planToJson(pObj, pJson); default: - assert(0); + // assert(0); break; } nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); @@ -1859,7 +2276,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { // case QUERY_NODE_LIMIT: // case QUERY_NODE_STATE_WINDOW: // case QUERY_NODE_SESSION_WINDOW: - // case QUERY_NODE_INTERVAL_WINDOW: + case QUERY_NODE_INTERVAL_WINDOW: + return jsonToIntervalWindowNode(pJson, pObj); case QUERY_NODE_NODE_LIST: return jsonToNodeListNode(pJson, pObj); // case QUERY_NODE_FILL: @@ -1877,20 +2295,24 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { // break; case QUERY_NODE_SELECT_STMT: return jsonToSelectStmt(pJson, pObj); - // case QUERY_NODE_LOGIC_PLAN_SCAN: - // return jsonToLogicScanNode(pJson, pObj); + case QUERY_NODE_CREATE_TOPIC_STMT: + return jsonToCreateTopicStmt(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_SCAN: + return jsonToLogicScanNode(pJson, pObj); // case QUERY_NODE_LOGIC_PLAN_JOIN: // return jsonToLogicJoinNode(pJson, pObj); // case QUERY_NODE_LOGIC_PLAN_AGG: // return jsonToLogicAggNode(pJson, pObj); - // case QUERY_NODE_LOGIC_PLAN_PROJECT: - // return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_PROJECT: + return jsonToLogicProjectNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: return jsonToPhysiTableScanNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: return jsonToPhysiStreamScanNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + return jsonToPhysiSysTableScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return jsonToPhysiProjectNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_JOIN: @@ -1899,14 +2321,16 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiAggNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return jsonToPhysiExchangeNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + return jsonToPhysiIntervalNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + return jsonToPhysiSessionWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return jsonToPhysiDispatchNode(pJson, pObj); case QUERY_NODE_PHYSICAL_SUBPLAN: return jsonToSubplan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN: return jsonToPlan(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: - return jsonToPhysiIntervalNode(pJson, pObj); default: assert(0); break; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index e3c2dff1208b427db0df79dfe1eafd2fb7682c2b..ff71c3bd585450fe2c544290cc35a625c42efc50 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -79,9 +79,14 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_STATE_WINDOW: res = walkNode(((SStateWindowNode*)pNode)->pCol, order, walker, pContext); break; - case QUERY_NODE_SESSION_WINDOW: - res = walkNode(((SSessionWindowNode*)pNode)->pCol, order, walker, pContext); + case QUERY_NODE_SESSION_WINDOW: { + SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; + res = walkNode(pSession->pCol, order, walker, pContext); + if (DEAL_RES_ERROR != res) { + res = walkNode(pSession->pGap, order, walker, pContext); + } break; + } case QUERY_NODE_INTERVAL_WINDOW: { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; res = walkNode(pInterval->pInterval, order, walker, pContext); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 0d48202f7339e8c9d77c7abb46c19c6a687bc688..5345e84cdb9635c7d961d09478164927048c1a90 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -22,7 +22,7 @@ #include "thash.h" static SNode* makeNode(ENodeType type, size_t size) { - SNode* p = calloc(1, size); + SNode* p = taosMemoryCalloc(1, size); if (NULL == p) { return NULL; } @@ -91,11 +91,9 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_CREATE_DATABASE_STMT: return makeNode(type, sizeof(SCreateDatabaseStmt)); case QUERY_NODE_DROP_DATABASE_STMT: - return makeNode(type, sizeof(SDropDatabaseStmt)); + return makeNode(type, sizeof(SDropDatabaseStmt)); case QUERY_NODE_ALTER_DATABASE_STMT: return makeNode(type, sizeof(SAlterDatabaseStmt)); - case QUERY_NODE_SHOW_DATABASES_STMT: - return makeNode(type, sizeof(SShowStmt)); case QUERY_NODE_CREATE_TABLE_STMT: return makeNode(type, sizeof(SCreateTableStmt)); case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: @@ -108,17 +106,12 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SDropTableStmt)); case QUERY_NODE_DROP_SUPER_TABLE_STMT: return makeNode(type, sizeof(SDropSuperTableStmt)); - case QUERY_NODE_SHOW_TABLES_STMT: - case QUERY_NODE_SHOW_STABLES_STMT: - return makeNode(type, sizeof(SShowStmt)); case QUERY_NODE_CREATE_USER_STMT: return makeNode(type, sizeof(SCreateUserStmt)); case QUERY_NODE_ALTER_USER_STMT: return makeNode(type, sizeof(SAlterUserStmt)); case QUERY_NODE_DROP_USER_STMT: return makeNode(type, sizeof(SDropUserStmt)); - case QUERY_NODE_SHOW_USERS_STMT: - return makeNode(type, sizeof(SShowStmt)); case QUERY_NODE_USE_DATABASE_STMT: return makeNode(type, sizeof(SUseDatabaseStmt)); case QUERY_NODE_CREATE_DNODE_STMT: @@ -127,12 +120,6 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SDropDnodeStmt)); case QUERY_NODE_ALTER_DNODE_STMT: return makeNode(type, sizeof(SAlterDnodeStmt)); - case QUERY_NODE_SHOW_DNODES_STMT: - return makeNode(type, sizeof(SShowStmt)); - case QUERY_NODE_SHOW_VGROUPS_STMT: - case QUERY_NODE_SHOW_MNODES_STMT: - case QUERY_NODE_SHOW_QNODES_STMT: - return makeNode(type, sizeof(SShowStmt)); case QUERY_NODE_CREATE_INDEX_STMT: return makeNode(type, sizeof(SCreateIndexStmt)); case QUERY_NODE_DROP_INDEX_STMT: @@ -145,6 +132,19 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SCreateTopicStmt)); case QUERY_NODE_DROP_TOPIC_STMT: return makeNode(type, sizeof(SDropTopicStmt)); + case QUERY_NODE_SHOW_DATABASES_STMT: + case QUERY_NODE_SHOW_TABLES_STMT: + case QUERY_NODE_SHOW_STABLES_STMT: + case QUERY_NODE_SHOW_USERS_STMT: + case QUERY_NODE_SHOW_DNODES_STMT: + case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_MNODES_STMT: + case QUERY_NODE_SHOW_MODULES_STMT: + case QUERY_NODE_SHOW_QNODES_STMT: + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + case QUERY_NODE_SHOW_INDEXES_STMT: + case QUERY_NODE_SHOW_STREAMS_STMT: + return makeNode(type, sizeof(SShowStmt)); case QUERY_NODE_LOGIC_PLAN_SCAN: return makeNode(type, sizeof(SScanLogicNode)); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -160,7 +160,7 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_LOGIC_PLAN_WINDOW: return makeNode(type, sizeof(SWindowLogicNode)); case QUERY_NODE_LOGIC_SUBPLAN: - return makeNode(type, sizeof(SSubLogicPlan)); + return makeNode(type, sizeof(SLogicSubplan)); case QUERY_NODE_LOGIC_PLAN: return makeNode(type, sizeof(SQueryLogicPlan)); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: @@ -171,6 +171,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(STableSeqScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: return makeNode(type, sizeof(SStreamScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + return makeNode(type, sizeof(SSystemTableScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_JOIN: @@ -183,6 +185,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + return makeNode(type, sizeof(SSessionWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return makeNode(type, sizeof(SDataDispatcherNode)); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -203,9 +207,9 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { case QUERY_NODE_VALUE: { SValueNode* pValue = (SValueNode*)*pNode; - tfree(pValue->literal); + taosMemoryFreeClear(pValue->literal); if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { - tfree(pValue->datum.p); + taosMemoryFreeClear(pValue->datum.p); } break; @@ -218,8 +222,8 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { break; case QUERY_NODE_REAL_TABLE: { SRealTableNode* pReal = (SRealTableNode*)*pNode; - tfree(pReal->pMeta); - tfree(pReal->pVgroupList); + taosMemoryFreeClear(pReal->pMeta); + taosMemoryFreeClear(pReal->pVgroupList); break; } case QUERY_NODE_TEMP_TABLE: @@ -258,8 +262,8 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { size_t size = taosArrayGetSize(pStmt->pDataBlocks); for (size_t i = 0; i < size; ++i) { SVgDataBlocks* pVg = taosArrayGetP(pStmt->pDataBlocks, i); - tfree(pVg->pData); - tfree(pVg); + taosMemoryFreeClear(pVg->pData); + taosMemoryFreeClear(pVg); } taosArrayDestroy(pStmt->pDataBlocks); break; @@ -288,7 +292,7 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { default: break; } - tfree(*pNode); + taosMemoryFreeClear(*pNode); return DEAL_RES_CONTINUE; } @@ -300,7 +304,7 @@ void nodesDestroyNode(SNodeptr pNode) { } SNodeList* nodesMakeList() { - SNodeList* p = calloc(1, sizeof(SNodeList)); + SNodeList* p = taosMemoryCalloc(1, sizeof(SNodeList)); if (NULL == p) { return NULL; } @@ -311,7 +315,7 @@ int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode) { if (NULL == pList || NULL == pNode) { return TSDB_CODE_SUCCESS; } - SListCell* p = calloc(1, sizeof(SListCell)); + SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; @@ -330,6 +334,7 @@ int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode) { int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode) { if (NULL == pNode) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = nodesListAppend(pList, pNode); @@ -339,6 +344,17 @@ int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode) { return code; } +int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode) { + if (NULL == *pList) { + *pList = nodesMakeList(); + if (NULL == *pList) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return nodesListAppend(*pList, pNode); +} + int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { if (NULL == pTarget || NULL == pSrc) { return TSDB_CODE_SUCCESS; @@ -354,7 +370,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { } pTarget->pTail = pSrc->pTail; pTarget->length += pSrc->length; - tfree(pSrc); + taosMemoryFreeClear(pSrc); return TSDB_CODE_SUCCESS; } @@ -379,7 +395,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { } SListCell* pNext = pCell->pNext; nodesDestroyNode(pCell->pNode); - tfree(pCell); + taosMemoryFreeClear(pCell); --(pList->length); return pNext; } @@ -403,7 +419,7 @@ void nodesDestroyList(SNodeList* pList) { while (NULL != pNext) { pNext = nodesListErase(pList, pNext); } - tfree(pList); + taosMemoryFreeClear(pList); } void nodesClearList(SNodeList* pList) { @@ -415,9 +431,9 @@ void nodesClearList(SNodeList* pList) { while (NULL != pNext) { SListCell* tmp = pNext; pNext = pNext->pNext; - tfree(tmp); + taosMemoryFreeClear(tmp); } - tfree(pList); + taosMemoryFreeClear(pList); } void* nodesGetValueFromNode(SValueNode *pNode) { diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 46bed1d61ed6136e1449833f41a0c2bba216631f..358edcb279f0ff7f7f328e7c86b6c5186a412885 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -84,9 +84,10 @@ SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode); SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode); SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); -SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableAlias, const SToken* pColumnName); +SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName); SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt); SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); @@ -94,12 +95,12 @@ SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNo SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); -SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias); +SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias); SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); -SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); @@ -119,9 +120,9 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt); SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt); SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal); -SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions); -SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName); -SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions); +SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions); +SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName); +SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions); SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt); SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal); @@ -141,17 +142,17 @@ SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName); SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName); SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal); -SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName); -SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDbName); -SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword); -SNode* createAlterUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, int8_t alterType, const SToken* pVal); -SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName); +SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); +SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbNamePattern); +SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword); +SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal); +SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode); SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue); -SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions); +SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions); SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding); -SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName); +SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName); SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName); diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index cbc35240a3c766fcc414733659042227dcd8b893..acd021572d4de499f85bbee35b5366c6d68e259c 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -78,6 +78,8 @@ typedef struct STableDataBlocks { char *pData; bool cloned; STagData tagData; + char tableName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; SParsedDataColInfo boundColumnInfo; SRowBuilder rowBuilder; @@ -115,10 +117,10 @@ static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t rowType, } } -static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { - pBlocks->tid = pTableMeta->suid; - pBlocks->uid = pTableMeta->uid; - pBlocks->sversion = pTableMeta->sversion; +static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows) { + pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? dataBuf->pTableMeta->uid : dataBuf->pTableMeta->suid); + pBlocks->uid = dataBuf->pTableMeta->uid; + pBlocks->sversion = dataBuf->pTableMeta->sversion; if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { return TSDB_CODE_TSC_INVALID_OPERATION; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 4d7a8e2a18b280e4f1882a0c96838c36d3f00959..171b406e1853f3d606adb5aebdaf51740594042a 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -23,6 +23,13 @@ extern "C" { #include "os.h" #include "query.h" +#define parserFatal(param, ...) qFatal("PARSER: " param, __VA_ARGS__) +#define parserError(param, ...) qError("PARSER: " param, __VA_ARGS__) +#define parserWarn(param, ...) qWarn("PARSER: " param, __VA_ARGS__) +#define parserInfo(param, ...) qInfo("PARSER: " param, __VA_ARGS__) +#define parserDebug(param, ...) qDebug("PARSER: " param, __VA_ARGS__) +#define parserTrace(param, ...) qTrace("PARSER: " param, __VA_ARGS__) + typedef struct SMsgBuf { int32_t len; char *buf; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 2298ae0761faaa2bce1d7a4186321299a22b8475..529fbb55c8e65521d8d3c20d26c963628750009f 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -83,14 +83,12 @@ cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); } cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); } cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } -cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } /************************************************ create/drop/alter/show dnode ****************************************/ cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); } cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } -cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, NULL); } cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B) NK_STRING(C). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, &C); } cmd ::= ALTER ALL DNODES NK_STRING(A). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, NULL); } @@ -112,12 +110,10 @@ cmd ::= ALTER LOCAL NK_STRING(A) NK_STRING(B). /************************************************ create/drop qnode ***************************************************/ cmd ::= CREATE QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &A); } cmd ::= DROP QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropQnodeStmt(pCxt, &A); } -cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL); } /************************************************ create/drop/show/use database ***************************************/ cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C). { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, A, &B, C); } cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } -cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); } cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } @@ -164,7 +160,7 @@ alter_db_option(A) ::= WAL NK_INTEGER(B). alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.val = B; } alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; } -/************************************************ create/drop/show table/stable ***************************************/ +/************************************************ create/drop table/stable ********************************************/ cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A); } @@ -172,8 +168,6 @@ cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); } -cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } -cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } cmd ::= ALTER TABLE alter_table_clause(A). { pCxt->pRootNode = A; } cmd ::= ALTER STABLE alter_table_clause(A). { pCxt->pRootNode = A; } @@ -286,6 +280,31 @@ col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C). col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } +/************************************************ show ****************************************************************/ +cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL, NULL); } +cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL, NULL); } +cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL, NULL); } +cmd ::= SHOW db_name_cond_opt(A) TABLES like_pattern_opt(B). { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, A, B); } +cmd ::= SHOW db_name_cond_opt(A) STABLES like_pattern_opt(B). { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, A, B); } +cmd ::= SHOW db_name_cond_opt(A) VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, A, NULL); } +cmd ::= SHOW MNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL, NULL); } +cmd ::= SHOW MODULES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MODULES_STMT, NULL, NULL); } +cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL, NULL); } +cmd ::= SHOW FUNCTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } +cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, A, B); } +cmd ::= SHOW STREAMS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } + +db_name_cond_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } +db_name_cond_opt(A) ::= db_name(B) NK_DOT. { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } + +like_pattern_opt(A) ::= . { A = NULL; } +like_pattern_opt(A) ::= LIKE NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } + +table_name_cond(A) ::= table_name(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } + +from_db_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } +from_db_opt(A) ::= FROM db_name(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } + %type func_name_list { SNodeList* } %destructor func_name_list { nodesDestroyList($$); } func_name_list(A) ::= func_name(B). { A = createNodeList(pCxt, B); } @@ -294,10 +313,11 @@ func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C). func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); } /************************************************ create index ********************************************************/ -cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); } -cmd ::= CREATE FULLTEXT INDEX - index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &A, &B, C, NULL); } -cmd ::= DROP INDEX index_name(A) ON table_name(B). { pCxt->pRootNode = createDropIndexStmt(pCxt, &A, &B); } +cmd ::= CREATE SMA INDEX not_exists_opt(D) + index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, &A, &B, NULL, C); } +cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D) + index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); } +cmd ::= DROP INDEX exists_opt(C) index_name(A) ON table_name(B). { pCxt->pRootNode = createDropIndexStmt(pCxt, C, &A, &B); } index_options(A) ::= . { A = NULL; } index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL @@ -317,13 +337,6 @@ cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C). cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS db_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C); } cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); } -/************************************************ show vgroups ********************************************************/ -cmd ::= SHOW VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); } -cmd ::= SHOW db_name(B) NK_DOT VGROUPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &B); } - -/************************************************ show mnodes *********************************************************/ -cmd ::= SHOW MNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL); } - /************************************************ select **************************************************************/ cmd ::= query_expression(A). { pCxt->pRootNode = A; } @@ -337,10 +350,31 @@ literal(A) ::= duration_literal(B). duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +signed(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } +signed(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } +signed(A) ::= NK_MINUS(B) NK_INTEGER(C). { + SToken t = B; + t.n = (C.z + C.n) - B.z; + A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + } +signed(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +signed(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +signed(A) ::= NK_MINUS(B) NK_FLOAT(C). { + SToken t = B; + t.n = (C.z + C.n) - B.z; + A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + } + +signed_literal(A) ::= signed(B). { A = B; } +signed_literal(A) ::= NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } +signed_literal(A) ::= NK_BOOL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } +signed_literal(A) ::= TIMESTAMP NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); } +signed_literal(A) ::= duration_literal(B). { A = releaseRawExprNode(pCxt, B); } + %type literal_list { SNodeList* } %destructor literal_list { nodesDestroyList($$); } -literal_list(A) ::= literal(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } -literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } +literal_list(A) ::= signed_literal(B). { A = createNodeList(pCxt, B); } +literal_list(A) ::= literal_list(B) NK_COMMA signed_literal(C). { A = addNodeToList(pCxt, B, C); } /************************************************ names and identifiers ***********************************************/ %type db_name { SToken } @@ -591,7 +625,7 @@ partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). twindow_clause_opt(A) ::= . { A = NULL; } twindow_clause_opt(A) ::= - SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); } + SESSION NK_LP column_reference(B) NK_COMMA duration_literal(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); } twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } twindow_clause_opt(A) ::= INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index eed6391d5c97af340b5827166fe4c00ba23cd21a..9630b0d68ce509248a61f01f6d7364b718e3279f 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -21,6 +21,7 @@ do { \ if (NULL == (p)) { \ pCxt->valid = false; \ + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); \ return NULL; \ } \ } while (0) @@ -330,113 +331,132 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { initSetTableOptionFp(); } -static bool checkUserName(SAstCreateContext* pCxt, const SToken* pUserName) { - if (NULL == pUserName) { - return false; +static void trimEscape(SToken* pName) { + if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) { + pName->z += 1; + pName->n -= 2; } - if (pUserName->n >= TSDB_USER_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); +} + +static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) { + if (NULL == pUserName) { pCxt->valid = false; + } else { + if (pUserName->n >= TSDB_USER_LEN) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); + pCxt->valid = false; + } } + trimEscape(pUserName); return pCxt->valid; } static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, char* pPassword) { if (NULL == pPasswordToken) { - return false; - } - if (pPasswordToken->n >= (TSDB_USET_PASSWORD_LEN - 2)) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); pCxt->valid = false; - return false; - } - strncpy(pPassword, pPasswordToken->z, pPasswordToken->n); - strdequote(pPassword); - if (strtrim(pPassword) <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); + } else if (pPasswordToken->n >= (TSDB_USET_PASSWORD_LEN - 2)) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); pCxt->valid = false; + } else { + strncpy(pPassword, pPasswordToken->z, pPasswordToken->n); + strdequote(pPassword); + if (strtrim(pPassword) <= 0) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); + pCxt->valid = false; + } } return pCxt->valid; } static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, char* pFqdn, int32_t* pPort) { if (NULL == pEp) { - return false; - } - if (pEp->n >= TSDB_FQDN_LEN + 2 + 6) { // format 'fqdn:port' - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; - } - char ep[TSDB_FQDN_LEN + 2 + 6]; - strncpy(ep, pEp->z, pEp->n); - strdequote(ep); - strtrim(ep); - char* pColon = strchr(ep, ':'); - if (NULL == pColon) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); pCxt->valid = false; - } - strncpy(pFqdn, ep, pColon - ep); - *pPort = strtol(pColon + 1, NULL, 10); - if (*pPort >= UINT16_MAX || *pPort <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); + } else if (pEp->n >= TSDB_FQDN_LEN + 2 + 6) { // format 'fqdn:port' + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); pCxt->valid = false; + } else { + char ep[TSDB_FQDN_LEN + 2 + 6]; + strncpy(ep, pEp->z, pEp->n); + strdequote(ep); + strtrim(ep); + char* pColon = strchr(ep, ':'); + if (NULL == pColon) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); + pCxt->valid = false; + } else { + strncpy(pFqdn, ep, pColon - ep); + *pPort = strtol(pColon + 1, NULL, 10); + if (*pPort >= UINT16_MAX || *pPort <= 0) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); + pCxt->valid = false; + } + } } return pCxt->valid; } static bool checkFqdn(SAstCreateContext* pCxt, const SToken* pFqdn) { if (NULL == pFqdn) { - return false; - } - if (pFqdn->n >= TSDB_FQDN_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); pCxt->valid = false; + } else { + if (pFqdn->n >= TSDB_FQDN_LEN) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); + pCxt->valid = false; + } } return pCxt->valid; } static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t* pPort) { if (NULL == pPortToken) { - return false; - } - *pPort = strtol(pPortToken->z, NULL, 10); - if (*pPort >= UINT16_MAX || *pPort <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); pCxt->valid = false; + } else { + *pPort = strtol(pPortToken->z, NULL, 10); + if (*pPort >= UINT16_MAX || *pPort <= 0) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); + pCxt->valid = false; + } } return pCxt->valid; } -static bool checkDbName(SAstCreateContext* pCxt, const SToken* pDbName) { +static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool query) { if (NULL == pDbName) { - return true; + pCxt->valid = (query ? NULL != pCxt->pQueryCxt->db : true); + } else { + pCxt->valid = pDbName->n < TSDB_DB_NAME_LEN ? true : false; } - pCxt->valid = pDbName->n < TSDB_DB_NAME_LEN ? true : false; + trimEscape(pDbName); return pCxt->valid; } -static bool checkTableName(SAstCreateContext* pCxt, const SToken* pTableName) { +static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) { if (NULL == pTableName) { - return true; + pCxt->valid = true; + } else { + pCxt->valid = pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; } - pCxt->valid = pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; + trimEscape(pTableName); return pCxt->valid; } -static bool checkColumnName(SAstCreateContext* pCxt, const SToken* pColumnName) { +static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) { if (NULL == pColumnName) { - return true; + pCxt->valid = true; + } else { + pCxt->valid = pColumnName->n < TSDB_COL_NAME_LEN ? true : false; } - pCxt->valid = pColumnName->n < TSDB_COL_NAME_LEN ? true : false; + trimEscape(pColumnName); return pCxt->valid; } -static bool checkIndexName(SAstCreateContext* pCxt, const SToken* pIndexName) { +static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { if (NULL == pIndexName) { - return false; + pCxt->valid = false; + } else { + pCxt->valid = pIndexName->n < TSDB_INDEX_NAME_LEN ? true : false; } - pCxt->valid = pIndexName->n < TSDB_INDEX_NAME_LEN ? true : false; + trimEscape(pIndexName); return pCxt->valid; } @@ -461,7 +481,7 @@ SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { CHECK_RAW_EXPR_NODE(pNode); SNode* tmp = ((SRawExprNode*)pNode)->pNode; - tfree(pNode); + taosMemoryFreeClear(pNode); return tmp; } @@ -491,7 +511,7 @@ SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode return pList; } -SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableAlias, const SToken* pColumnName) { +SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName) { if (!checkTableName(pCxt, pTableAlias) || !checkColumnName(pCxt, pColumnName)) { return NULL; } @@ -508,9 +528,12 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); CHECK_OUT_OF_MEM(val); val->literal = strndup(pLiteral->z, pLiteral->n); + if (TK_NK_ID != pLiteral->type && (IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) { + trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n); + } CHECK_OUT_OF_MEM(val->literal); val->node.resType.type = dataType; - val->node.resType.bytes = tDataTypes[dataType].bytes; + val->node.resType.bytes = IS_VAR_DATA_TYPE(dataType) ? strlen(val->literal) : tDataTypes[dataType].bytes; if (TSDB_DATA_TYPE_TIMESTAMP == dataType) { val->node.resType.precision = TSDB_TIME_PRECISION_MILLI; } @@ -532,6 +555,23 @@ SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) return (SNode*)val; } +SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt) { + if (NULL == pCxt->pQueryCxt->db) { + return NULL; + } + + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + val->literal = strdup(pCxt->pQueryCxt->db); + CHECK_OUT_OF_MEM(val->literal); + val->isDuration = false; + val->translate = false; + val->node.resType.type = TSDB_DATA_TYPE_BINARY; + val->node.resType.bytes = strlen(val->literal); + val->node.resType.precision = TSDB_TIME_PRECISION_MILLI; + return (SNode*)val; +} + SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2) { SLogicConditionNode* cond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); CHECK_OUT_OF_MEM(cond); @@ -576,8 +616,8 @@ SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList) { return (SNode*)list; } -SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias) { - if (!checkDbName(pCxt, pDbName) || !checkTableName(pCxt, pTableName)) { +SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias) { + if (!checkDbName(pCxt, pDbName, true) || !checkTableName(pCxt, pTableName) || !checkTableName(pCxt, pTableAlias)) { return NULL; } SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); @@ -593,6 +633,9 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const strncpy(realTable->table.tableAlias, pTableName->z, pTableName->n); } strncpy(realTable->table.tableName, pTableName->z, pTableName->n); + if (NULL != pCxt->pQueryCxt->db) { + strcpy(realTable->useDbName, pCxt->pQueryCxt->db); + } return (SNode*)realTable; } @@ -636,11 +679,11 @@ SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order return (SNode*)orderByExpr; } -SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal) { +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap) { SSessionWindowNode* session = (SSessionWindowNode*)nodesMakeNode(QUERY_NODE_SESSION_WINDOW); CHECK_OUT_OF_MEM(session); session->pCol = pCol; - // session->gap = getInteger(pVal); + session->pGap = pGap; return (SNode*)session; } @@ -811,8 +854,8 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal); } -SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions) { - if (!checkDbName(pCxt, pDbName)) { +SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) { + if (!checkDbName(pCxt, pDbName, false)) { return NULL; } SCreateDatabaseStmt* pStmt = (SCreateDatabaseStmt*)nodesMakeNode(QUERY_NODE_CREATE_DATABASE_STMT); @@ -823,8 +866,8 @@ SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, cons return (SNode*)pStmt; } -SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName) { - if (!checkDbName(pCxt, pDbName)) { +SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName) { + if (!checkDbName(pCxt, pDbName, false)) { return NULL; } SDropDatabaseStmt* pStmt = (SDropDatabaseStmt*)nodesMakeNode(QUERY_NODE_DROP_DATABASE_STMT); @@ -834,8 +877,8 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, con return (SNode*)pStmt; } -SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions) { - if (!checkDbName(pCxt, pDbName)) { +SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions) { + if (!checkDbName(pCxt, pDbName, false)) { return NULL; } SAlterDatabaseStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DATABASE_STMT); @@ -892,7 +935,7 @@ SDataType createDataType(uint8_t type) { } SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { - SDataType dt = { .type = type, .precision = 0, .scale = 0, .bytes = tDataTypes[type].bytes }; + SDataType dt = { .type = type, .precision = 0, .scale = 0, .bytes = strtol(pLen->z, NULL, 10) }; return dt; } @@ -1003,28 +1046,34 @@ SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const return (SNode*)pStmt; } -SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) { +SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { + if (!checkDbName(pCxt, pDbName, false)) { + return NULL; + } SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT); CHECK_OUT_OF_MEM(pStmt); strncpy(pStmt->dbName, pDbName->z, pDbName->n); return (SNode*)pStmt; } -SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDbName) { - if (!checkDbName(pCxt, pDbName)) { +static bool needDbShowStmt(ENodeType type) { + return QUERY_NODE_SHOW_TABLES_STMT == type || QUERY_NODE_SHOW_STABLES_STMT == type || QUERY_NODE_SHOW_VGROUPS_STMT == type; +} + +SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbNamePattern) { + if (needDbShowStmt(type) && NULL == pDbName && NULL == pCxt->pQueryCxt->db) { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "db not specified"); + pCxt->valid = false; return NULL; } SShowStmt* pStmt = nodesMakeNode(type);; CHECK_OUT_OF_MEM(pStmt); - if (NULL != pDbName) { - strncpy(pStmt->dbName, pDbName->z, pDbName->n); - } else if (NULL != pCxt->pQueryCxt->db) { - strcpy(pStmt->dbName, pCxt->pQueryCxt->db); - } + pStmt->pDbName = pDbName; + pStmt->pTbNamePattern = pTbNamePattern; return (SNode*)pStmt; } -SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword) { +SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword) { char password[TSDB_USET_PASSWORD_LEN] = {0}; if (!checkUserName(pCxt, pUserName) || !checkPassword(pCxt, pPassword, password)) { return NULL; @@ -1036,7 +1085,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, co return (SNode*)pStmt; } -SNode* createAlterUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, int8_t alterType, const SToken* pVal) { +SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal) { if (!checkUserName(pCxt, pUserName)) { return NULL; } @@ -1055,7 +1104,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, int return (SNode*)pStmt; } -SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName) { +SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName) { if (!checkUserName(pCxt, pUserName)) { return NULL; } @@ -1111,13 +1160,14 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const return (SNode*)pStmt; } -SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions) { +SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions) { if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { return NULL; } SCreateIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->indexType = type; + pStmt->ignoreExists = ignoreExists; strncpy(pStmt->indexName, pIndexName->z, pIndexName->n); strncpy(pStmt->tableName, pTableName->z, pTableName->n); pStmt->pCols = pCols; @@ -1135,12 +1185,13 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt return (SNode*)pOptions; } -SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName) { +SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName) { if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { return NULL; } SDropIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); + pStmt->ignoreNotExists = ignoreNotExists; strncpy(pStmt->indexName, pIndexName->z, pIndexName->n); strncpy(pStmt->tableName, pTableName->z, pTableName->n); return (SNode*)pStmt; diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index e992f150aae875eb2620dcd5f40c347054f9f820..5c38ccaff80cfa553c5d6ed75ed3ebb6400483e6 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -29,7 +29,7 @@ extern void ParseTrace(FILE*, char*); int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); - void *pParser = ParseAlloc(malloc); + void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc); int32_t i = 0; while (1) { SToken t0 = {0}; @@ -73,9 +73,9 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { } abort_parse: - ParseFree(pParser, free); + ParseFree(pParser, (FFree)taosMemoryFree); if (cxt.valid) { - *pQuery = calloc(1, sizeof(SQuery)); + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index a37820634f9c8cc80d5cce61047d502facd0a8d2..b2fc39d0645bbf30839a08eae24b20f62c3307ff 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -28,6 +28,13 @@ pSql += index; \ } while (0) +#define NEXT_TOKEN_WITH_PREV(pSql, sToken) \ + do { \ + int32_t index = 0; \ + sToken = tStrGetToken(pSql, &index, true); \ + pSql += index; \ + } while (0) + #define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \ do { \ sToken = tStrGetToken(pSql, &index, false); \ @@ -41,15 +48,12 @@ } \ } while (0) -enum { - TSDB_USE_SERVER_TS = 0, - TSDB_USE_CLI_TS = 1, -}; - typedef struct SInsertParseContext { SParseContext* pComCxt; // input char *pSql; // input SMsgBuf msg; // input + char dbFName[TSDB_DB_FNAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; STableMeta* pTableMeta; // each table SParsedDataColInfo tags; // each table SKVRowBuilder tagsBuilder; // each table @@ -228,6 +232,9 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); pCxt->pTableMeta->vgId = vg.vgId; // todo remove + strcpy(pCxt->tableName, name.tname); + tNameGetFullDbName(&name, pCxt->dbFName); + return TSDB_CODE_SUCCESS; } @@ -241,7 +248,7 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS return -1; } -static void buildMsgHeader(SVgDataBlocks* blocks) { +static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { SSubmitReq* submit = (SSubmitReq*)blocks->pData; submit->header.vgId = htonl(blocks->vg.vgId); submit->header.contLen = htonl(blocks->size); @@ -252,7 +259,7 @@ static void buildMsgHeader(SVgDataBlocks* blocks) { while (numOfBlocks--) { int32_t dataLen = blk->dataLen; blk->uid = htobe64(blk->uid); - blk->tid = htonl(blk->tid); + blk->suid = htobe64(blk->suid); blk->padding = htonl(blk->padding); blk->sversion = htonl(blk->sversion); blk->dataLen = htonl(blk->dataLen); @@ -270,7 +277,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { } for (size_t i = 0; i < numOfVg; ++i) { STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); - SVgDataBlocks* dst = calloc(1, sizeof(SVgDataBlocks)); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -278,7 +285,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { dst->numOfTables = src->numOfTables; dst->size = src->size; TSWAP(dst->pData, src->pData, char*); - buildMsgHeader(dst); + buildMsgHeader(src, dst); taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); } return TSDB_CODE_SUCCESS; @@ -291,20 +298,7 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) } TSKEY k = *(TSKEY *)start; - - if (k == INT64_MIN) { - if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { - return TSDB_CODE_FAILED; // client time/server time can not be mixed - } - pDataBlocks->tsSource = TSDB_USE_SERVER_TS; - } else { - if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { - return TSDB_CODE_FAILED; // client time/server time can not be mixed - } - pDataBlocks->tsSource = TSDB_USE_CLI_TS; - } - - if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { + if (k <= pDataBlocks->prevTS) { pDataBlocks->ordered = false; } @@ -352,7 +346,7 @@ static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time sToken = tStrGetToken(pTokenEnd, &index, false); pTokenEnd += index; - if (sToken.type == TK_MINUS || sToken.type == TK_NK_PLUS) { + if (sToken.type == TK_NK_MINUS || sToken.type == TK_NK_PLUS) { index = 0; valueToken = tStrGetToken(pTokenEnd, &index, false); pTokenEnd += index; @@ -391,7 +385,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha } // Remove quotation marks - if (TSDB_DATA_TYPE_BINARY == type) { + if (TK_NK_STRING == pToken->type) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } @@ -453,7 +447,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return func(&tmpVal, pSchema->bytes, param); } - return func(getNullValue(pSchema->type), 0, param); + return func(NULL, 0, param); } switch (pSchema->type) { @@ -626,9 +620,13 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } varDataSetLen(rowEnd, output); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, true, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); + if (value == NULL) { // it is a null data + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); + } else { + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); + } } return TSDB_CODE_SUCCESS; } @@ -686,7 +684,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; if (!isOrdered) { - pColList->colIdxInfo = calloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); if (NULL == pColList->colIdxInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -748,7 +746,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, SToken sToken; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { - NEXT_TOKEN(pCxt->pSql, sToken); + NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; param.schema = pSchema; CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); @@ -762,7 +760,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, // todo construct payload - tfree(row); + taosMemoryFreeClear(row); return 0; } @@ -814,7 +812,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, SToken sToken = {0}; // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { - NEXT_TOKEN(pCxt->pSql, sToken); + NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); SSchema *pSchema = &schema[spd->boundedColumns[i] - 1]; param.schema = pSchema; getMemRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); @@ -895,7 +893,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows)); SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); - if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows)) { + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than 32767"); } @@ -905,7 +903,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da } static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { - tfree(pCxt->pTableMeta); + taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); tdDestroyKVRowBuilder(&pCxt->tagsBuilder); } @@ -915,16 +913,16 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { return; } - tfree(pDataBlock->pData); + taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); + taosMemoryFreeClear(pDataBlock->pTableMeta); } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } - tfree(pDataBlock); + taosMemoryFreeClear(pDataBlock); } static void destroyInsertParseContext(SInsertParseContext* pCxt) { @@ -972,7 +970,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { STableDataBlocks *dataBuf = NULL; CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); - + strcpy(dataBuf->tableName, pCxt->tableName); + strcpy(dataBuf->dbFName, pCxt->dbFName); + if (TK_NK_LP == sToken.type) { // pSql -> field1_name, ...) CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); @@ -1026,11 +1026,10 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { }; if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pOutput) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY; } - *pQuery = calloc(1, sizeof(SQuery)); + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1045,6 +1044,5 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { code = parseInsertBody(&context); } destroyInsertParseContext(&context); - terrno = code; return code; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index da5a652018f7066b7f176462e5ad633cfb834bd1..e516053b1e9b62c5b8ce62b09fd157b818c718c6 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -47,8 +47,8 @@ void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t pColList->numOfCols = numOfCols; pColList->numOfBound = numOfCols; pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundedColumns = calloc(pColList->numOfCols, sizeof(int32_t)); - pColList->cols = calloc(pColList->numOfCols, sizeof(SBoundColumn)); + pColList->boundedColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(int32_t)); + pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); pColList->colIdxInfo = NULL; pColList->flen = 0; pColList->allNullLen = 0; @@ -103,14 +103,14 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { } void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { - tfree(pColList->boundedColumns); - tfree(pColList->cols); - tfree(pColList->colIdxInfo); + taosMemoryFreeClear(pColList->boundedColumns); + taosMemoryFreeClear(pColList->cols); + taosMemoryFreeClear(pColList->colIdxInfo); } static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); + STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -123,9 +123,9 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->nAllocSize = dataBuf->headerSize * 2; } - dataBuf->pData = malloc(dataBuf->nAllocSize); + dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { - tfree(dataBuf); + taosMemoryFreeClear(dataBuf); return TSDB_CODE_TSC_OUT_OF_MEMORY; } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); @@ -141,7 +141,6 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->prevTS = INT64_MIN; dataBuf->rowSize = rowSize; dataBuf->size = startOffset; - dataBuf->tsSource = -1; dataBuf->vgId = dataBuf->pTableMeta->vgId; assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); @@ -191,16 +190,16 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { return; } - tfree(pDataBlock->pData); + taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); + taosMemoryFreeClear(pDataBlock->pTableMeta); } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } - tfree(pDataBlock); + taosMemoryFreeClear(pDataBlock); } void destroyBlockArrayList(SArray* pDataBlockList) { @@ -284,7 +283,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey // allocate memory size_t nAlloc = nRows * sizeof(SBlockKeyTuple); if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char *tmp = realloc(pBlkKeyInfo->pKeyTuple, nAlloc); + char *tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -434,7 +433,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return ret; } @@ -445,14 +444,14 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); + char* tmp = taosMemoryRealloc(dataBuf->pData, dataBuf->nAllocSize); if (tmp != NULL) { dataBuf->pData = tmp; } else { // failed to allocate memory, free already allocated memory and return error code taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(dataBuf->pData); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } @@ -463,8 +462,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(dataBuf->pData); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return code; } ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); @@ -493,7 +492,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t // free the table data blocks; taosHashCleanup(pVnodeDataBlockHashList); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); *pVgDataBlocks = pVnodeDataBlockList; return TSDB_CODE_SUCCESS; } @@ -510,7 +509,7 @@ int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t remain = pDataBlock->nAllocSize - pDataBlock->size; } - char *tmp = realloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); + char *tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); if (tmp != NULL) { pDataBlock->pData = tmp; memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 4549660c762f13e93ffaff5aec7c6a6712604d30..79651cd325fe6c9582ab875a6bc713689c8699cd 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -61,12 +61,14 @@ static SKeyword keywordTable[] = { {"FROM", TK_FROM}, {"FSYNC", TK_FSYNC}, {"FUNCTION", TK_FUNCTION}, + {"FUNCTIONS", TK_FUNCTIONS}, {"GROUP", TK_GROUP}, {"HAVING", TK_HAVING}, {"IF", TK_IF}, {"IMPORT", TK_IMPORT}, {"IN", TK_IN}, {"INDEX", TK_INDEX}, + {"INDEXES", TK_INDEXES}, {"INNER", TK_INNER}, {"INT", TK_INT}, {"INSERT", TK_INSERT}, @@ -85,6 +87,7 @@ static SKeyword keywordTable[] = { {"MINROWS", TK_MINROWS}, {"MINUS", TK_MINUS}, {"MNODES", TK_MNODES}, + {"MODULES", TK_MODULES}, {"NCHAR", TK_NCHAR}, {"NMATCH", TK_NMATCH}, {"NONE", TK_NONE}, @@ -116,6 +119,7 @@ static SKeyword keywordTable[] = { {"STABLE", TK_STABLE}, {"STABLES", TK_STABLES}, {"STATE_WINDOW", TK_STATE_WINDOW}, + {"STREAMS", TK_STREAMS}, {"STREAM_MODE", TK_STREAM_MODE}, {"TABLE", TK_TABLE}, {"TABLES", TK_TABLES}, @@ -161,10 +165,8 @@ static SKeyword keywordTable[] = { // {"UPLUS", TK_UPLUS}, // {"BITNOT", TK_BITNOT}, // {"ACCOUNTS", TK_ACCOUNTS}, - // {"MODULES", TK_MODULES}, // {"QUERIES", TK_QUERIES}, // {"CONNECTIONS", TK_CONNECTIONS}, - // {"STREAMS", TK_STREAMS}, // {"VARIABLES", TK_VARIABLES}, // {"SCORES", TK_SCORES}, // {"GRANTS", TK_GRANTS}, @@ -234,7 +236,6 @@ static SKeyword keywordTable[] = { // {"TOPICS", TK_TOPICS}, // {"COMPACT", TK_COMPACT}, // {"MODIFY", TK_MODIFY}, - // {"FUNCTIONS", TK_FUNCTIONS}, // {"OUTPUTTYPE", TK_OUTPUTTYPE}, // {"AGGREGATE", TK_AGGREGATE}, // {"BUFSIZE", TK_BUFSIZE}, @@ -266,10 +267,10 @@ static void doInitKeywordsTable(void) { } } -static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT; +static TdThreadOnce keywordsHashTableInit = PTHREAD_ONCE_INIT; static int32_t tKeywordCode(const char* z, int n) { - pthread_once(&keywordsHashTableInit, doInitKeywordsTable); + taosThreadOnce(&keywordsHashTableInit, doInitKeywordsTable); char key[512] = {0}; if (n > tListLen(key)) { // too long token, can not be any other token type @@ -320,7 +321,7 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { *tokenId = TK_NK_COMMENT; return i; } - *tokenId = TK_MINUS; + *tokenId = TK_NK_MINUS; return 1; } case '(': { @@ -593,7 +594,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) { int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src); SToken ntoken; - *str = calloc(1, size); + *str = taosMemoryCalloc(1, size); strncpy(*str, src, bsize); strcat(*str, newToken); @@ -602,7 +603,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) { ntoken.n = (uint32_t)nsize; ntoken.z = *str + bsize; - tfree(src); + taosMemoryFreeClear(src); return ntoken; } @@ -674,7 +675,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { } else { // support parse the -/+number format - if ((isPrevOptr) && (t0.type == TK_MINUS || t0.type == TK_NK_PLUS)) { + if ((isPrevOptr) && (t0.type == TK_NK_MINUS || t0.type == TK_NK_PLUS)) { len = tGetToken(&str[*i + t0.n], &type); if (type == TK_NK_INTEGER || type == TK_NK_FLOAT) { t0.type = type; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 171ec81ca20078cf657cf9a9cbc8092cc2db2a45..1c721f3caf8846a981dc91c12ec27be8619b64a4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -30,8 +30,14 @@ typedef struct STranslateContext { ESqlClause currClause; SSelectStmt* pCurrStmt; SCmdMsgInfo* pCmdMsg; + SHashObj* pDbs; + SHashObj* pTables; } STranslateContext; +typedef struct SFullDatabaseName { + char fullDbName[TSDB_DB_FNAME_LEN]; +} SFullDatabaseName; + static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode); @@ -70,14 +76,125 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { return TSDB_CODE_SUCCESS; } -static SName* toName(int32_t acctId, const SRealTableNode* pRealTable, SName* pName) { +static SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; - strcpy(pName->dbname, pRealTable->table.dbName); - strcpy(pName->tname, pRealTable->table.tableName); + strcpy(pName->dbname, pDbName); + strcpy(pName->tname, pTableName); return pName; } +static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { + SFullDatabaseName name = {0}; + strcpy(name.fullDbName, pFullDbName); + return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName)); +} + +static int32_t collectUseDatabase(const SName* pName, SHashObj* pDbs) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pName, dbFName); + return collectUseDatabaseImpl(dbFName, pDbs); +} + +static int32_t collectUseTable(const SName* pName, SHashObj* pDbs) { + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pName, fullName); + return taosHashPut(pDbs, fullName, strlen(fullName), pName, sizeof(SName)); +} + +static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STableMeta** pMeta) { + SParseContext* pParCxt = pCxt->pParseCxt; + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } + if (TSDB_CODE_SUCCESS == code) { + code = catalogGetTableMeta(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pMeta); + } + if (TSDB_CODE_SUCCESS != code) { + parserError("catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname, pName->tname); + } + return code; +} + +static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { + SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; + strcpy(name.dbname, pDbName); + strcpy(name.tname, pTableName); + return getTableMetaImpl(pCxt, &name, pMeta); +} + +static int32_t getTableDistVgInfo(STranslateContext* pCxt, const SName* pName, SArray** pVgInfo) { + SParseContext* pParCxt = pCxt->pParseCxt; + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } + if (TSDB_CODE_SUCCESS == code) { + code = catalogGetTableDistVgInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pVgInfo); + } + if (TSDB_CODE_SUCCESS != code) { + parserError("catalogGetTableDistVgInfo error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname, pName->tname); + } + return code; +} + +static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArray** pVgInfo) { + SParseContext* pParCxt = pCxt->pParseCxt; + char fullDbName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pName, fullDbName); + int32_t code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = catalogGetDBVgInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, fullDbName, pVgInfo); + } + if (TSDB_CODE_SUCCESS != code) { + parserError("catalogGetDBVgInfo error, code:%s, dbFName:%s", tstrerror(code), fullDbName); + } + return code; +} + +static int32_t getDBVgInfo(STranslateContext* pCxt, const char* pDbName, SArray** pVgInfo) { + SName name; + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + return getDBVgInfoImpl(pCxt, &name, pVgInfo); +} + +static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pName, SVgroupInfo* pInfo) { + SParseContext* pParCxt = pCxt->pParseCxt; + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } + if (TSDB_CODE_SUCCESS == code) { + code = catalogGetTableHashVgroup(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pInfo); + } + if (TSDB_CODE_SUCCESS != code) { + parserError("catalogGetTableHashVgroup error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname, pName->tname); + } + return code; +} + +static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SVgroupInfo* pInfo) { + SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; + strcpy(name.dbname, pDbName); + strcpy(name.tname, pTableName); + return getTableHashVgroupImpl(pCxt, &name, pInfo); +} + +static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, int32_t* pTableNum) { + SParseContext* pParCxt = pCxt->pParseCxt; + int32_t code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum); + } + if (TSDB_CODE_SUCCESS != code) { + parserError("catalogGetDBVgVersion error, code:%s, dbFName:%s", tstrerror(code), pDbFName); + } + return code; +} + static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { int cmp = 0; if ('\0' != pCol->dbName[0]) { @@ -244,18 +361,9 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { return found ? DEAL_RES_CONTINUE : translateColumnWithoutPrefix(pCxt, pCol); } -static int32_t trimStringWithVarFormat(const char* src, int32_t len, bool format, char* dst) { - char* dstVal = dst; - if (format) { - varDataSetLen(dst, len); - dstVal = varDataVal(dst); - } - return trimString(src, len, dstVal, len); -} - static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { if (pVal->isDuration) { - if (parseAbsoluteDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) { + if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } } else { @@ -290,26 +398,18 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - int32_t n = strlen(pVal->literal); - pVal->datum.p = calloc(1, n + VARSTR_HEADER_SIZE); + pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); } - trimStringWithVarFormat(pVal->literal, n, true, pVal->datum.p); + varDataSetLen(pVal->datum.p, pVal->node.resType.bytes); + strncpy(varDataVal(pVal->datum.p), pVal->literal, pVal->node.resType.bytes); break; } case TSDB_DATA_TYPE_TIMESTAMP: { - int32_t n = strlen(pVal->literal); - char* tmp = calloc(1, n); - if (NULL == tmp) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); - } - int32_t len = trimStringWithVarFormat(pVal->literal, n, false, tmp); - if (taosParseTime(tmp, &pVal->datum.i, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { - tfree(tmp); + if (taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } - tfree(tmp); break; } case TSDB_DATA_TYPE_JSON: @@ -498,42 +598,71 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) return TSDB_CODE_SUCCESS; } -static int32_t setTableVgroupList(SParseContext* pCxt, SName* name, SRealTableNode* pRealTable) { - if (pCxt->streamQuery) { +static int32_t toVgroupsInfo(SArray* pVgs, SVgroupsInfo** pVgsInfo) { + size_t vgroupNum = taosArrayGetSize(pVgs); + *pVgsInfo = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum); + if (NULL == *pVgsInfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (*pVgsInfo)->numOfVgroups = vgroupNum; + for (int32_t i = 0; i < vgroupNum; ++i) { + SVgroupInfo *vg = taosArrayGet(pVgs, i); + (*pVgsInfo)->vgroups[i] = *vg; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + // todo release + if (0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = TSDB_CODE_SUCCESS; + SArray* vgroupList = NULL; + if ('\0' != pRealTable->useDbName[0]) { + code = getDBVgInfo(pCxt, pRealTable->useDbName, &vgroupList); + } else { + code = getDBVgInfoImpl(pCxt, pName, &vgroupList); + } + + if (TSDB_CODE_SUCCESS == code) { + // todo remove + //if (NULL != vgroupList && taosArrayGetSize(vgroupList) > 0 && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { + // taosArrayPopTailBatch(vgroupList, taosArrayGetSize(vgroupList) - 1); + //} + + code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); + } + taosArrayDestroy(vgroupList); + + return code; +} + +static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + if (pCxt->pParseCxt->topicQuery) { return TSDB_CODE_SUCCESS; } + int32_t code = TSDB_CODE_SUCCESS; if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { SArray* vgroupList = NULL; - int32_t code = catalogGetTableDistVgInfo(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, name, &vgroupList); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - size_t vgroupNum = taosArrayGetSize(vgroupList); - pRealTable->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum); - if (NULL == pRealTable->pVgroupList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pRealTable->pVgroupList->numOfVgroups = vgroupNum; - for (int32_t i = 0; i < vgroupNum; ++i) { - SVgroupInfo *vg = taosArrayGet(vgroupList, i); - pRealTable->pVgroupList->vgroups[i] = *vg; + code = getTableDistVgInfo(pCxt, pName, &vgroupList); + if (TSDB_CODE_SUCCESS == code) { + code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); } - taosArrayDestroy(vgroupList); + } else if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { + code = setSysTableVgroupList(pCxt, pName, pRealTable); } else { - pRealTable->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); if (NULL == pRealTable->pVgroupList) { return TSDB_CODE_OUT_OF_MEMORY; } pRealTable->pVgroupList->numOfVgroups = 1; - int32_t code = catalogGetTableHashVgroup(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, name, pRealTable->pVgroupList->vgroups); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups); } - return TSDB_CODE_SUCCESS; + return code; } static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { @@ -542,12 +671,12 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { case QUERY_NODE_REAL_TABLE: { SRealTableNode* pRealTable = (SRealTableNode*)pTable; SName name; - code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &(pCxt->pParseCxt->mgmtEpSet), - toName(pCxt->pParseCxt->acctId, pRealTable, &name), &(pRealTable->pMeta)); + code = getTableMetaImpl(pCxt, + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), &(pRealTable->pMeta)); if (TSDB_CODE_SUCCESS != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName); } - code = setTableVgroupList(pCxt->pParseCxt, &name, pRealTable); + code = setTableVgroupList(pCxt, &name, pRealTable); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -710,11 +839,30 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SNodeList* pGroupByList return translateExprList(pCxt, pGroupByList); } +static int32_t translateIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { + SValueNode* pIntervalVal = (SValueNode*)pInterval->pInterval; + SValueNode* pIntervalOffset = (SValueNode*)pInterval->pOffset; + SValueNode* pSliding = (SValueNode*)pInterval->pSliding; + if (pIntervalVal->datum.i <= 0) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL, pIntervalVal->literal); + } + return TSDB_CODE_SUCCESS; +} + static int32_t doTranslateWindow(STranslateContext* pCxt, SNode* pWindow) { + switch (nodeType(pWindow)) { + case QUERY_NODE_INTERVAL_WINDOW: + return translateIntervalWindow(pCxt, (SIntervalWindowNode*)pWindow); + default: + break; + } return TSDB_CODE_SUCCESS; } static int32_t translateWindow(STranslateContext* pCxt, SNode* pWindow) { + if (NULL == pWindow) { + return TSDB_CODE_SUCCESS; + } pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, pWindow); if (TSDB_CODE_SUCCESS == code) { @@ -799,14 +947,14 @@ static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseS SCreateDbReq createReq = {0}; buildCreateDbReq(pCxt, pStmt, &createReq); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DB; pCxt->pCmdMsg->msgLen = tSerializeSCreateDbReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -822,14 +970,14 @@ static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* tNameGetFullDbName(&name, dropReq.db); dropReq.ignoreNotExists = pStmt->ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_DB; pCxt->pCmdMsg->msgLen = tSerializeSDropDbReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -857,14 +1005,14 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm SAlterDbReq alterReq = {0}; buildAlterDbReq(pCxt, pStmt, &alterReq); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_DB; pCxt->pCmdMsg->msgLen = tSerializeSAlterDbReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -873,12 +1021,22 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm return TSDB_CODE_SUCCESS; } +static int32_t calcTypeBytes(SDataType dt) { + if (TSDB_DATA_TYPE_BINARY == dt.type) { + return dt.bytes + VARSTR_HEADER_SIZE; + } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { + return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } else { + return dt.bytes; + } +} + static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) { *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); SNode* pNode; FOREACH(pNode, pList) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; - SField field = { .type = pCol->dataType.type, .bytes = pCol->dataType.bytes }; + SField field = { .type = pCol->dataType.type, .bytes = calcTypeBytes(pCol->dataType) }; strcpy(field.name, pCol->colName); taosArrayPush(*pArray, &field); } @@ -898,7 +1056,7 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt strcpy(tableName.tname, pStmt->tableName); tNameExtractFullName(&tableName, createReq.name); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { tFreeSMCreateStbReq(&createReq); return TSDB_CODE_OUT_OF_MEMORY; @@ -906,7 +1064,7 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_STB; pCxt->pCmdMsg->msgLen = tSerializeSMCreateStbReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { tFreeSMCreateStbReq(&createReq); return TSDB_CODE_OUT_OF_MEMORY; @@ -922,14 +1080,14 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p tNameExtractFullName(pTableName, dropReq.name); dropReq.igNotExists = ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_STB; pCxt->pCmdMsg->msgLen = tSerializeSMDropStbReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -941,11 +1099,10 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt) { SDropTableClause* pClause = nodesListGetNode(pStmt->pTables, 0); - SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; - strcpy(tableName.dbname, pClause->dbName); - strcpy(tableName.tname, pClause->tableName); STableMeta* pTableMeta = NULL; - int32_t code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &(pCxt->pParseCxt->mgmtEpSet), &tableName, &pTableMeta); + SName tableName; + int32_t code = getTableMetaImpl( + pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta); if (TSDB_CODE_SUCCESS == code) { if (TSDB_SUPER_TABLE == pTableMeta->tableType) { code = doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); @@ -953,8 +1110,8 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt // todo : drop normal table or child table code = TSDB_CODE_FAILED; } + taosMemoryFreeClear(pTableMeta); } - tfree(pTableMeta); return code; } @@ -1017,14 +1174,14 @@ static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pSt } } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_STB; pCxt->pCmdMsg->msgLen = tSerializeSMAlterStbReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1034,22 +1191,23 @@ static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pSt } static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) { + SUseDbReq usedbReq = {0}; SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - - SUseDbReq usedbReq = {0}; tNameExtractFullName(&name, usedbReq.db); + int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable); + if (TSDB_CODE_SUCCESS != code) { + return code; + } - catalogGetDBVgVersion(pCxt->pParseCxt->pCatalog, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable); - - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_USE_DB; pCxt->pCmdMsg->msgLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1065,14 +1223,14 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt createReq.superUser = 0; strcpy(createReq.pass, pStmt->password); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_USER; pCxt->pCmdMsg->msgLen = tSerializeSCreateUserReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1091,14 +1249,14 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt strcpy(alterReq.dbname, pCxt->pParseCxt->db); } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_USER; pCxt->pCmdMsg->msgLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1111,14 +1269,14 @@ static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) SDropUserReq dropReq = {0}; strcpy(dropReq.user, pStmt->useName); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_USER; pCxt->pCmdMsg->msgLen = tSerializeSDropUserReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1132,14 +1290,14 @@ static int32_t translateCreateDnode(STranslateContext* pCxt, SCreateDnodeStmt* p strcpy(createReq.fqdn, pStmt->fqdn); createReq.port = pStmt->port; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1154,14 +1312,14 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt strcpy(dropReq.fqdn, pStmt->fqdn); dropReq.port = pStmt->port; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1176,14 +1334,14 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt strcpy(cfgReq.config, pStmt->config); strcpy(cfgReq.value, pStmt->value); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CONFIG_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1216,21 +1374,21 @@ static int32_t nodeTypeToShowType(ENodeType nt) { static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) { SShowReq showReq = { .type = nodeTypeToShowType(nodeType(pStmt)) }; - if ('\0' != pStmt->dbName[0]) { - SName name = {0}; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); - char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, showReq.db); - } - - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + // if ('\0' != pStmt->dbName[0]) { + // SName name = {0}; + // tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + // char dbFname[TSDB_DB_FNAME_LEN] = {0}; + // tNameGetFullDbName(&name, showReq.db); + // } + + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_SHOW; pCxt->pCmdMsg->msgLen = tSerializeSShowReq(NULL, 0, &showReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1240,25 +1398,17 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) { } static int32_t translateShowTables(STranslateContext* pCxt) { - SName name = {0}; - SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); - if (pCxt->pParseCxt->db == NULL || strlen(pCxt->pParseCxt->db) == 0) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_INVALID_OPERATION, "db not specified"); - } - - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, strlen(pCxt->pParseCxt->db)); - char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, dbFname); + SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq)); SArray* array = NULL; - int32_t code = catalogGetDBVgInfo(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &pCxt->pParseCxt->mgmtEpSet, dbFname, false, &array); - if (code != TSDB_CODE_SUCCESS) { + int32_t code = getDBVgInfo(pCxt, pCxt->pParseCxt->db, &array); + if (TSDB_CODE_SUCCESS != code) { return code; } SVgroupInfo* info = taosArrayGet(array, 0); pShowReq->head.vgId = htonl(info->vgId); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1271,52 +1421,127 @@ static int32_t translateShowTables(STranslateContext* pCxt) { return TSDB_CODE_SUCCESS; } -static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { - SVCreateTSmaReq createSmaReq = {0}; +static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, char* pTableName, int32_t* pVgId) { + SVgroupInfo vg = {0}; + int32_t code = getTableHashVgroup(pCxt, pCxt->pParseCxt->db, pTableName, &vg); + if (TSDB_CODE_SUCCESS == code) { + *pVgId = vg.vgId; + } + return code; +} - if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval) || - (NULL != pStmt->pOptions->pOffset && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset)) || - (NULL != pStmt->pOptions->pSliding && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding))) { - return pCxt->errCode; +static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLen) { + *pSql = strdup(pCxt->pParseCxt->pSql); + if (NULL == *pSql) { + return TSDB_CODE_OUT_OF_MEMORY; } + *pLen = pCxt->pParseCxt->sqlLen + 1; + return TSDB_CODE_SUCCESS; +} - createSmaReq.tSma.intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; - createSmaReq.tSma.slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : 0); - strcpy(createSmaReq.tSma.indexName, pStmt->indexName); +static int32_t getSmaIndexExpr(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pExpr, int32_t* pLen) { + return nodesListToString(pStmt->pOptions->pFuncs, false, pExpr, pLen); +} - SName name; - name.type = TSDB_TABLE_NAME_T; - name.acctId = pCxt->pParseCxt->acctId; +static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) { + SSelectStmt* pSelect = nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == pSelect) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); + if (NULL == pTable) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pTable->table.dbName, pCxt->pParseCxt->db); + strcpy(pTable->table.tableName, pStmt->tableName); + pSelect->pFromTable = (SNode*)pTable; + + pSelect->pProjectionList = nodesCloneList(pStmt->pOptions->pFuncs); + if (NULL == pTable) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SIntervalWindowNode* pInterval = nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); + if (NULL == pInterval) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + pSelect->pWindow = (SNode*)pInterval; + pInterval->pInterval = nodesCloneNode(pStmt->pOptions->pInterval); + pInterval->pOffset = nodesCloneNode(pStmt->pOptions->pOffset); + pInterval->pSliding = nodesCloneNode(pStmt->pOptions->pSliding); + if (NULL == pInterval->pInterval || (NULL != pStmt->pOptions->pOffset && NULL == pInterval->pOffset) || + (NULL != pStmt->pOptions->pSliding && NULL == pInterval->pSliding)) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = translateQuery(pCxt, (SNode*)pSelect); + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pSelect, false, pAst, pLen); + } + nodesDestroyNode(pSelect); + return code; +} + +static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) { + SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; strcpy(name.dbname, pCxt->pParseCxt->db); + strcpy(name.tname, pStmt->indexName); + tNameExtractFullName(&name, pReq->name); strcpy(name.tname, pStmt->tableName); - STableMeta* pMeta = NULL; - int32_t code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &pCxt->pParseCxt->mgmtEpSet, &name, &pMeta); - if (TSDB_CODE_SUCCESS != code) { - return code; + name.tname[strlen(pStmt->tableName)] = '\0'; + tNameExtractFullName(&name, pReq->stb); + pReq->igExists = pStmt->ignoreExists; + pReq->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; + pReq->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; + pReq->offset = (NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0); + pReq->sliding = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pReq->interval); + pReq->slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pReq->intervalUnit); + + int32_t code = getSmaIndexDstVgId(pCxt, pStmt->tableName, &pReq->dstVgId); + if (TSDB_CODE_SUCCESS == code) { + code = getSmaIndexSql(pCxt, &pReq->sql, &pReq->sqlLen); + } + if (TSDB_CODE_SUCCESS == code) { + code = getSmaIndexExpr(pCxt, pStmt, &pReq->expr, &pReq->exprLen); + } + if (TSDB_CODE_SUCCESS == code) { + code = getSmaIndexBuildAst(pCxt, pStmt, &pReq->ast, &pReq->astLen); + } + + return code; +} + +static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval) || + (NULL != pStmt->pOptions->pOffset && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset)) || + (NULL != pStmt->pOptions->pSliding && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding))) { + return pCxt->errCode; } - createSmaReq.tSma.tableUid = pMeta->uid; - createSmaReq.tSma.interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; - createSmaReq.tSma.sliding = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : 0); - code = nodesListToString(pStmt->pOptions->pFuncs, false, &createSmaReq.tSma.expr, &createSmaReq.tSma.exprLen); + SMCreateSmaReq createSmaReq = {0}; + int32_t code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); if (TSDB_CODE_SUCCESS != code) { return code; } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; - pCxt->pCmdMsg->msgType = TDMT_VND_CREATE_SMA; - pCxt->pCmdMsg->msgLen = tSerializeSVCreateTSmaReq(NULL, &createSmaReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_SMA; + pCxt->pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, &createSmaReq); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } - void* pBuf = pCxt->pCmdMsg->pMsg; - tSerializeSVCreateTSmaReq(&pBuf, &createSmaReq); - tdDestroyTSma(&createSmaReq.tSma); + tSerializeSMCreateSmaReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createSmaReq); + tFreeSMCreateSmaReq(&createSmaReq); return TSDB_CODE_SUCCESS; } @@ -1334,14 +1559,14 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt SVDropTSmaReq dropSmaReq = {0}; strcpy(dropSmaReq.indexName, pStmt->indexName); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA; pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1354,14 +1579,14 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* pStmt) { SMCreateQnodeReq createReq = { .dnodeId = pStmt->dnodeId }; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_DND_CREATE_QNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1373,14 +1598,14 @@ static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* p static int32_t translateDropQnode(STranslateContext* pCxt, SDropQnodeStmt* pStmt) { SDDropQnodeReq dropReq = { .dnodeId = pStmt->dnodeId }; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_DND_DROP_QNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1393,7 +1618,7 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p SCMCreateTopicReq createReq = {0}; if (NULL != pStmt->pQuery) { - pCxt->pParseCxt->streamQuery = true; + pCxt->pParseCxt->topicQuery = true; int32_t code = translateQuery(pCxt, pStmt->pQuery); if (TSDB_CODE_SUCCESS == code) { code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL); @@ -1416,14 +1641,14 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p tNameExtractFullName(&name, createReq.name); createReq.igExists = pStmt->ignoreExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_TOPIC; pCxt->pCmdMsg->msgLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1442,14 +1667,14 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt tNameExtractFullName(&name, dropReq.name); dropReq.igNotExists = pStmt->ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_TOPIC; pCxt->pCmdMsg->msgLen = tSerializeSMDropTopicReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1561,24 +1786,27 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { return code; } -static int32_t setReslutSchema(STranslateContext* pCxt, SQuery* pQuery) { - if (QUERY_NODE_SELECT_STMT == nodeType(pQuery->pRoot)) { - SSelectStmt* pSelect = (SSelectStmt*)pQuery->pRoot; - pQuery->numOfResCols = LIST_LENGTH(pSelect->pProjectionList); - pQuery->pResSchema = calloc(pQuery->numOfResCols, sizeof(SSchema)); - if (NULL == pQuery->pResSchema) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); +int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + SSelectStmt* pSelect = (SSelectStmt*) pRoot; + *numOfCols = LIST_LENGTH(pSelect->pProjectionList); + *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); + if (NULL == (*pSchema)) { + return TSDB_CODE_OUT_OF_MEMORY; } + SNode* pNode; int32_t index = 0; FOREACH(pNode, pSelect->pProjectionList) { SExprNode* pExpr = (SExprNode*)pNode; - pQuery->pResSchema[index].type = pExpr->resType.type; - pQuery->pResSchema[index].bytes = pExpr->resType.bytes; - strcpy(pQuery->pResSchema[index].name, pExpr->aliasName); + (*pSchema)[index].type = pExpr->resType.type; + (*pSchema)[index].bytes = pExpr->resType.bytes; + (*pSchema)[index].colId = index + 1; + strcpy((*pSchema)[index].name, pExpr->aliasName); index +=1; } } + return TSDB_CODE_SUCCESS; } @@ -1592,14 +1820,159 @@ static void destroyTranslateContext(STranslateContext* pCxt) { } if (NULL != pCxt->pCmdMsg) { - tfree(pCxt->pCmdMsg->pMsg); - tfree(pCxt->pCmdMsg); + taosMemoryFreeClear(pCxt->pCmdMsg->pMsg); + taosMemoryFreeClear(pCxt->pCmdMsg); + } + + taosHashCleanup(pCxt->pDbs); + taosHashCleanup(pCxt->pTables); +} + +static const char* getSysTableName(ENodeType type) { + switch (type) { + case QUERY_NODE_SHOW_DATABASES_STMT: + return TSDB_INS_TABLE_USER_DATABASES; + case QUERY_NODE_SHOW_TABLES_STMT: + return TSDB_INS_TABLE_USER_TABLES; + case QUERY_NODE_SHOW_STABLES_STMT: + return TSDB_INS_TABLE_USER_STABLES; + case QUERY_NODE_SHOW_USERS_STMT: + return TSDB_INS_TABLE_USER_USERS; + case QUERY_NODE_SHOW_DNODES_STMT: + return TSDB_INS_TABLE_DNODES; + case QUERY_NODE_SHOW_VGROUPS_STMT: + return TSDB_INS_TABLE_VGROUPS; + case QUERY_NODE_SHOW_MNODES_STMT: + return TSDB_INS_TABLE_MNODES; + case QUERY_NODE_SHOW_MODULES_STMT: + return TSDB_INS_TABLE_MODULES; + case QUERY_NODE_SHOW_QNODES_STMT: + return TSDB_INS_TABLE_QNODES; + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + return TSDB_INS_TABLE_USER_FUNCTIONS; + case QUERY_NODE_SHOW_INDEXES_STMT: + return TSDB_INS_TABLE_USER_INDEXES; + case QUERY_NODE_SHOW_STREAMS_STMT: + return TSDB_INS_TABLE_USER_STREAMS; + default: + break; + } + return NULL; +} + +static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) { + SSelectStmt* pSelect = nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == pSelect) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); + if (NULL == pTable) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; } + strcpy(pTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB); + strcpy(pTable->table.tableName, getSysTableName(showType)); + pSelect->pFromTable = (SNode*)pTable; + + *pStmt = pSelect; + + return TSDB_CODE_SUCCESS; +} + +static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SNode* pRight, SNode** pOp) { + if (NULL == pRight) { + return TSDB_CODE_SUCCESS; + } + + SOperatorNode* pOper = nodesMakeNode(QUERY_NODE_OPERATOR); + if (NULL == pOper) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pOper->opType = opType; + pOper->pLeft = nodesMakeNode(QUERY_NODE_COLUMN); + pOper->pRight = nodesCloneNode(pRight); + if (NULL == pOper->pLeft || NULL == pOper->pRight) { + nodesDestroyNode(pOper); + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(((SColumnNode*)pOper->pLeft)->colName, pColName); + + *pOp = (SNode*)pOper; + return TSDB_CODE_SUCCESS; +} + +static const char* getTbNameColName(ENodeType type) { + return (QUERY_NODE_SHOW_STABLES_STMT == type ? "stable_name" : "table_name"); +} + +static int32_t createLogicCondNode(SNode* pCond1, SNode* pCond2, SNode** pCond) { + SLogicConditionNode* pCondition = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pCondition) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCondition->condType = LOGIC_COND_TYPE_AND; + pCondition->pParameterList = nodesMakeList(); + if (NULL == pCondition->pParameterList) { + nodesDestroyNode(pCondition); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS != nodesListAppend(pCondition->pParameterList, pCond1) || + TSDB_CODE_SUCCESS != nodesListAppend(pCondition->pParameterList, pCond2)) { + nodesDestroyNode(pCondition); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pCond = (SNode*)pCondition; + return TSDB_CODE_SUCCESS; +} + +static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) { + SNode* pDbCond = NULL; + SNode* pTbCond = NULL; + if (TSDB_CODE_SUCCESS != createOperatorNode(OP_TYPE_EQUAL, "db_name", pShow->pDbName, &pDbCond) || + TSDB_CODE_SUCCESS != createOperatorNode(OP_TYPE_LIKE, getTbNameColName(nodeType(pShow)), pShow->pTbNamePattern, &pTbCond)) { + nodesDestroyNode(pDbCond); + nodesDestroyNode(pTbCond); + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (NULL != pDbCond && NULL != pTbCond) { + if (TSDB_CODE_SUCCESS != createLogicCondNode(pDbCond, pTbCond, &pSelect->pWhere)) { + nodesDestroyNode(pDbCond); + nodesDestroyNode(pTbCond); + return TSDB_CODE_OUT_OF_MEMORY; + } + } else { + pSelect->pWhere = (NULL == pDbCond ? pTbCond : pDbCond); + } + + if (NULL != pShow->pDbName) { + strcpy(((SRealTableNode*)pSelect->pFromTable)->useDbName, ((SValueNode*)pShow->pDbName)->literal); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t rewriteShow(STranslateContext* pCxt, SQuery* pQuery) { + SSelectStmt* pStmt = NULL; + int32_t code = createSelectStmtForShow(nodeType(pQuery->pRoot), &pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = createShowCondition((SShowStmt*)pQuery->pRoot, pStmt); + } + if (TSDB_CODE_SUCCESS == code) { + pQuery->showRewrite = true; + nodesDestroyNode(pQuery->pRoot); + pQuery->pRoot = (SNode*)pStmt; + } + return code; } typedef struct SVgroupTablesBatch { SVCreateTbBatchReq req; SVgroupInfo info; + char dbName[TSDB_DB_NAME_LEN]; } SVgroupTablesBatch; static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema) { @@ -1610,17 +1983,24 @@ static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema } static void destroyCreateTbReq(SVCreateTbReq* pReq) { - tfree(pReq->name); - tfree(pReq->ntbCfg.pSchema); + taosMemoryFreeClear(pReq->dbFName); + taosMemoryFreeClear(pReq->name); + taosMemoryFreeClear(pReq->ntbCfg.pSchema); } -static int32_t buildNormalTableBatchReq( - const char* pTableName, const SNodeList* pColumns, const SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) { +static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, const char* pTableName, + const SNodeList* pColumns, const SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + SName name = { .type = TSDB_DB_NAME_T, .acctId = acctId }; + strcpy(name.dbname, pDbName); + tNameGetFullDbName(&name, dbFName); + SVCreateTbReq req = {0}; req.type = TD_NORMAL_TABLE; + req.dbFName = strdup(dbFName); req.name = strdup(pTableName); req.ntbCfg.nCols = LIST_LENGTH(pColumns); - req.ntbCfg.pSchema = calloc(req.ntbCfg.nCols, sizeof(SSchema)); + req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntbCfg.pSchema) { destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; @@ -1633,6 +2013,7 @@ static int32_t buildNormalTableBatchReq( } pBatch->info = *pVgroupInfo; + strcpy(pBatch->dbName, pDbName); pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); if (NULL == pBatch->req.pArray) { destroyCreateTbReq(&req); @@ -1645,7 +2026,7 @@ static int32_t buildNormalTableBatchReq( static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) { int tlen = sizeof(SMsgHead) + tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1654,7 +2035,7 @@ static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tSerializeSVCreateTbBatchReq(&pBuf, &(pTbBatch->req)); - SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks)); + SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == pVgData) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1671,25 +2052,19 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { size_t size = taosArrayGetSize(pTbBatch->req.pArray); for(int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); - tfree(pTableReq->name); + taosMemoryFreeClear(pTableReq->dbFName); + taosMemoryFreeClear(pTableReq->name); if (pTableReq->type == TSDB_NORMAL_TABLE) { - tfree(pTableReq->ntbCfg.pSchema); + taosMemoryFreeClear(pTableReq->ntbCfg.pSchema); } else if (pTableReq->type == TSDB_CHILD_TABLE) { - tfree(pTableReq->ctbCfg.pTag); + taosMemoryFreeClear(pTableReq->ctbCfg.pTag); } } taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t getTableHashVgroup(SParseContext* pCxt, const char* pDbName, const char* pTableName, SVgroupInfo* pInfo) { - SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->acctId }; - strcpy(name.dbname, pDbName); - strcpy(name.tname, pTableName); - return catalogGetTableHashVgroup(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, &name, pInfo); -} - static int32_t rewriteToVnodeModifOpStmt(SQuery* pQuery, SArray* pBufArray) { SVnodeModifOpStmt* pNewStmt = nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); if (pNewStmt == NULL) { @@ -1706,20 +2081,20 @@ static void destroyCreateTbReqArray(SArray* pArray) { size_t size = taosArrayGetSize(pArray); for (size_t i = 0; i < size; ++i) { SVgDataBlocks* pVg = taosArrayGetP(pArray, i); - tfree(pVg->pData); - tfree(pVg); + taosMemoryFreeClear(pVg->pData); + taosMemoryFreeClear(pVg); } taosArrayDestroy(pArray); } -static int32_t buildCreateTableDataBlock(const SCreateTableStmt* pStmt, const SVgroupInfo* pInfo, SArray** pBufArray) { +static int32_t buildCreateTableDataBlock(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pInfo, SArray** pBufArray) { *pBufArray = taosArrayInit(1, POINTER_BYTES); if (NULL == *pBufArray) { return TSDB_CODE_OUT_OF_MEMORY; } SVgroupTablesBatch tbatch = {0}; - int32_t code = buildNormalTableBatchReq(pStmt->tableName, pStmt->pCols, pInfo, &tbatch); + int32_t code = buildNormalTableBatchReq(acctId, pStmt->dbName, pStmt->tableName, pStmt->pCols, pInfo, &tbatch); if (TSDB_CODE_SUCCESS == code) { code = serializeVgroupTablesBatch(&tbatch, *pBufArray); } @@ -1735,10 +2110,10 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { SCreateTableStmt* pStmt = (SCreateTableStmt*)pQuery->pRoot; SVgroupInfo info = {0}; - int32_t code = getTableHashVgroup(pCxt->pParseCxt, pStmt->dbName, pStmt->tableName, &info); + int32_t code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info); SArray* pBufArray = NULL; if (TSDB_CODE_SUCCESS == code) { - code = buildCreateTableDataBlock(pStmt, &info, &pBufArray); + code = buildCreateTableDataBlock(pCxt->pParseCxt->acctId, pStmt, &info, &pBufArray); } if (TSDB_CODE_SUCCESS == code) { code = rewriteToVnodeModifOpStmt(pQuery, pBufArray); @@ -1750,9 +2125,16 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { return code; } -static void addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const char* pTableName, SKVRow row, uint64_t suid, SVgroupInfo* pVgInfo) { +static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, + const char* pDbName, const char* pTableName, SKVRow row, uint64_t suid, SVgroupInfo* pVgInfo) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + SName name = { .type = TSDB_DB_NAME_T, .acctId = acctId }; + strcpy(name.dbname, pDbName); + tNameGetFullDbName(&name, dbFName); + struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; + req.dbFName = strdup(dbFName); req.name = strdup(pTableName); req.ctbCfg.suid = suid; req.ctbCfg.pTag = row; @@ -1761,6 +2143,7 @@ static void addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const char* pTabl if (pTableBatch == NULL) { SVgroupTablesBatch tBatch = {0}; tBatch.info = *pVgInfo; + strcpy(tBatch.dbName, pDbName); tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); taosArrayPush(tBatch.req.pArray, &req); @@ -1873,12 +2256,9 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt, SHashObj* pVgroupHashmap) { - SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; - strcpy(name.dbname, pStmt->useDbName); - strcpy(name.tname, pStmt->useTableName); STableMeta* pSuperTableMeta = NULL; - int32_t code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &pCxt->pParseCxt->mgmtEpSet, &name, &pSuperTableMeta); - + int32_t code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta); + SKVRowBuilder kvRowBuilder = {0}; if (TSDB_CODE_SUCCESS == code) { code = tdInitKVRowBuilder(&kvRowBuilder); @@ -1904,18 +2284,18 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla SVgroupInfo info = {0}; if (TSDB_CODE_SUCCESS == code) { - code = getTableHashVgroup(pCxt->pParseCxt, pStmt->dbName, pStmt->tableName, &info); + code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info); } if (TSDB_CODE_SUCCESS == code) { - addCreateTbReqIntoVgroup(pVgroupHashmap, pStmt->tableName, row, pSuperTableMeta->uid, &info); + addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt->dbName, pStmt->tableName, row, pSuperTableMeta->uid, &info); } - tfree(pSuperTableMeta); + taosMemoryFreeClear(pSuperTableMeta); tdDestroyKVRowBuilder(&kvRowBuilder); return code; } -static SArray* serializeVgroupsTablesBatch(SHashObj* pVgroupHashmap) { +static SArray* serializeVgroupsTablesBatch(int32_t acctId, SHashObj* pVgroupHashmap) { SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); if (NULL == pBufArray) { return NULL; @@ -1954,7 +2334,7 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) } } - SArray* pBufArray = serializeVgroupsTablesBatch(pVgroupHashmap); + SArray* pBufArray = serializeVgroupsTablesBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); taosHashCleanup(pVgroupHashmap); if (NULL == pBufArray) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1971,6 +2351,20 @@ static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pQuery->pRoot)) { + case QUERY_NODE_SHOW_DATABASES_STMT: + case QUERY_NODE_SHOW_TABLES_STMT: + case QUERY_NODE_SHOW_STABLES_STMT: + case QUERY_NODE_SHOW_USERS_STMT: + case QUERY_NODE_SHOW_DNODES_STMT: + case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_MNODES_STMT: + case QUERY_NODE_SHOW_MODULES_STMT: + case QUERY_NODE_SHOW_QNODES_STMT: + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + case QUERY_NODE_SHOW_INDEXES_STMT: + case QUERY_NODE_SHOW_STREAMS_STMT: + code = rewriteShow(pCxt, pQuery); + break; case QUERY_NODE_CREATE_TABLE_STMT: if (NULL == ((SCreateTableStmt*)pQuery->pRoot)->pTags) { code = rewriteCreateTable(pCxt, pQuery); @@ -1997,7 +2391,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { pQuery->haveResultSet = true; pQuery->directRpc = false; pQuery->msgType = TDMT_VND_QUERY; - code = setReslutSchema(pCxt, pQuery); + code = qExtractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema); break; case QUERY_NODE_VNODE_MODIF_STMT: pQuery->haveResultSet = false; @@ -2012,6 +2406,31 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { pQuery->msgType = pQuery->pCmdMsg->msgType; break; } + + if (NULL != pCxt->pDbs) { + pQuery->pDbList = taosArrayInit(taosHashGetSize(pCxt->pDbs), TSDB_DB_FNAME_LEN); + if (NULL == pQuery->pDbList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SFullDatabaseName* pDb = taosHashIterate(pCxt->pDbs, NULL); + while (NULL != pDb) { + taosArrayPush(pQuery->pDbList, pDb->fullDbName); + pDb = taosHashIterate(pCxt->pDbs, pDb); + } + } + + if (NULL != pCxt->pTables) { + pQuery->pTableList = taosArrayInit(taosHashGetSize(pCxt->pTables), sizeof(SName)); + if (NULL == pQuery->pTableList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SName* pTable = taosHashIterate(pCxt->pTables, NULL); + while (NULL != pTable) { + taosArrayPush(pQuery->pTableList, pTable); + pTable = taosHashIterate(pCxt->pTables, pTable); + } + } + return code; } @@ -2022,8 +2441,13 @@ int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { .msgBuf = { .buf = pParseCxt->pMsg, .len = pParseCxt->msgLen }, .pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES), .currLevel = 0, - .currClause = 0 + .currClause = 0, + .pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK), + .pTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK) }; + if (NULL == cxt.pNsLevel) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t code = fmFuncMgtInit(); if (TSDB_CODE_SUCCESS == code) { code = rewriteQuery(&cxt, pQuery); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 69c2380b4538765a68881722f5926d4500f4a28b..80d04c5ee4f98ea1c09fedba4c1690bd7fceabef 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -59,6 +59,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Endpoint should be in the format of 'fqdn:port'"; case TSDB_CODE_PAR_EXPRIE_STATEMENT: return "This statement is no longer supported"; + case TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL: + return "This interval value is too small : %s"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -71,6 +73,7 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...) { va_start(vArgList, errCode); vsnprintf(pBuf->buf, pBuf->len, getSyntaxErrFormat(errCode), vArgList); va_end(vArgList); + terrno = errCode; return errCode; } @@ -119,7 +122,7 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); size_t size = getTableMetaSize(pTableMeta); - STableMeta* p = malloc(size); + STableMeta* p = taosMemoryMalloc(size); memcpy(p, pTableMeta, size); return p; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 868bd755208ed1ac0cdafe3774f4d979dc3d197a..92bd6f111b03ccf7b9b3232eefae50a934d61032 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -38,11 +38,14 @@ static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) { } int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery) { + int32_t code = TSDB_CODE_SUCCESS; if (isInsertSql(pCxt->pSql, pCxt->sqlLen)) { - return parseInsertSql(pCxt, pQuery); + code = parseInsertSql(pCxt, pQuery); } else { - return parseSqlIntoAst(pCxt, pQuery); + code = parseSqlIntoAst(pCxt, pQuery); } + terrno = code; + return code; } void qDestroyQuery(SQuery* pQueryNode) { @@ -50,10 +53,10 @@ void qDestroyQuery(SQuery* pQueryNode) { return; } nodesDestroyNode(pQueryNode->pRoot); - tfree(pQueryNode->pResSchema); + taosMemoryFreeClear(pQueryNode->pResSchema); if (NULL != pQueryNode->pCmdMsg) { - tfree(pQueryNode->pCmdMsg->pMsg); - tfree(pQueryNode->pCmdMsg); + taosMemoryFreeClear(pQueryNode->pCmdMsg->pMsg); + taosMemoryFreeClear(pQueryNode->pCmdMsg); } - tfree(pQueryNode); + taosMemoryFreeClear(pQueryNode); } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 3d88eb5336342371f8942305bcb78add301d91e8..451587c8bf1f42481b3947a5c07b3576a5ed7a84 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -73,7 +73,7 @@ ** which is ParseTOKENTYPE. The entry in the union ** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() +** zero the stack is dynamically sized using taosMemoryRealloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument ** ParseARG_PDECL A parameter declaration for the %extra_argument ** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter @@ -98,25 +98,25 @@ # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned char -#define YYNOCODE 249 +#define YYCODETYPE unsigned short int +#define YYNOCODE 258 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SNode* yy26; - EOrder yy32; - SNodeList* yy64; - EOperatorType yy80; - bool yy107; - EFillMode yy192; - SToken yy353; - SDataType yy370; - EJoinType yy372; - ENullOrder yy391; - SAlterOption yy443; - int32_t yy448; + ENullOrder yy73; + SNodeList* yy136; + SNode* yy140; + EJoinType yy144; + SToken yy149; + EOrder yy158; + int32_t yy160; + SAlterOption yy233; + SDataType yy256; + EFillMode yy306; + EOperatorType yy320; + bool yy497; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -131,17 +131,17 @@ typedef union { #define ParseCTX_PARAM #define ParseCTX_FETCH #define ParseCTX_STORE -#define YYNSTATE 414 -#define YYNRULE 316 -#define YYNTOKEN 160 -#define YY_MAX_SHIFT 413 -#define YY_MIN_SHIFTREDUCE 631 -#define YY_MAX_SHIFTREDUCE 946 -#define YY_ERROR_ACTION 947 -#define YY_ACCEPT_ACTION 948 -#define YY_NO_ACTION 949 -#define YY_MIN_REDUCE 950 -#define YY_MAX_REDUCE 1265 +#define YYNSTATE 430 +#define YYNRULE 337 +#define YYNTOKEN 163 +#define YY_MAX_SHIFT 429 +#define YY_MIN_SHIFTREDUCE 662 +#define YY_MAX_SHIFTREDUCE 998 +#define YY_ERROR_ACTION 999 +#define YY_ACCEPT_ACTION 1000 +#define YY_NO_ACTION 1001 +#define YY_MIN_REDUCE 1002 +#define YY_MAX_REDUCE 1338 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -208,379 +208,398 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1182) +#define YY_ACTTAB_COUNT (1263) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 996, 347, 347, 24, 162, 334, 253, 1057, 227, 1049, - /* 10 */ 108, 1115, 1158, 31, 29, 27, 26, 25, 951, 331, - /* 20 */ 347, 1098, 1060, 1060, 103, 64, 962, 31, 29, 27, - /* 30 */ 26, 25, 263, 233, 1051, 973, 1106, 1108, 304, 76, - /* 40 */ 346, 1060, 75, 74, 73, 72, 71, 70, 69, 68, - /* 50 */ 67, 201, 398, 397, 396, 395, 394, 393, 392, 391, - /* 60 */ 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, - /* 70 */ 380, 1130, 1130, 30, 28, 829, 31, 29, 27, 26, - /* 80 */ 25, 224, 346, 808, 76, 852, 42, 75, 74, 73, - /* 90 */ 72, 71, 70, 69, 68, 67, 275, 102, 270, 42, - /* 100 */ 806, 274, 1244, 1056, 273, 1063, 271, 1105, 88, 272, - /* 110 */ 289, 12, 950, 214, 201, 1243, 1055, 815, 1103, 1242, - /* 120 */ 23, 222, 346, 847, 848, 849, 850, 851, 853, 855, - /* 130 */ 856, 857, 807, 254, 1, 10, 86, 85, 84, 83, - /* 140 */ 82, 81, 80, 79, 78, 1132, 290, 303, 852, 747, - /* 150 */ 369, 368, 367, 751, 366, 753, 754, 365, 756, 362, - /* 160 */ 410, 762, 359, 764, 765, 356, 353, 287, 1244, 215, - /* 170 */ 942, 943, 817, 310, 305, 10, 913, 1143, 809, 812, - /* 180 */ 285, 116, 1130, 23, 222, 1242, 847, 848, 849, 850, - /* 190 */ 851, 853, 855, 856, 857, 117, 117, 1158, 300, 911, - /* 200 */ 912, 914, 915, 1132, 331, 31, 29, 27, 26, 25, - /* 210 */ 333, 347, 347, 888, 1130, 1143, 344, 345, 815, 319, - /* 220 */ 105, 1025, 60, 1144, 1147, 1183, 948, 230, 249, 200, - /* 230 */ 1179, 117, 1060, 1060, 347, 1158, 972, 292, 248, 64, - /* 240 */ 1130, 1244, 318, 247, 315, 246, 269, 309, 333, 379, - /* 250 */ 1158, 96, 1130, 1197, 116, 1060, 180, 331, 1242, 1090, - /* 260 */ 61, 1144, 1147, 1183, 243, 91, 1105, 217, 1179, 111, - /* 270 */ 1194, 237, 229, 1130, 245, 244, 308, 1103, 1143, 406, - /* 280 */ 405, 158, 30, 28, 889, 89, 347, 296, 1210, 819, - /* 290 */ 224, 175, 808, 1244, 317, 112, 1190, 1191, 1158, 1195, - /* 300 */ 228, 315, 27, 26, 25, 318, 116, 1060, 102, 806, - /* 310 */ 1242, 333, 51, 1143, 239, 1130, 1062, 58, 1202, 884, - /* 320 */ 12, 818, 91, 61, 1144, 1147, 1183, 92, 1036, 1053, - /* 330 */ 217, 1179, 111, 1158, 1052, 31, 29, 27, 26, 25, - /* 340 */ 331, 807, 89, 1, 887, 267, 333, 232, 1143, 266, - /* 350 */ 1130, 1211, 113, 1190, 1191, 102, 1195, 910, 61, 1144, - /* 360 */ 1147, 1183, 235, 1062, 117, 217, 1179, 1256, 1158, 410, - /* 370 */ 102, 77, 372, 971, 268, 331, 1217, 254, 1062, 1143, - /* 380 */ 1105, 333, 30, 28, 157, 1130, 234, 809, 812, 1035, - /* 390 */ 224, 1103, 808, 61, 1144, 1147, 1183, 829, 347, 1158, - /* 400 */ 217, 1179, 1256, 236, 9, 8, 331, 30, 28, 806, - /* 410 */ 1130, 1240, 333, 30, 28, 224, 1130, 808, 1143, 1060, - /* 420 */ 12, 224, 334, 808, 61, 1144, 1147, 1183, 1116, 1197, - /* 430 */ 808, 217, 1179, 1256, 806, 669, 970, 670, 1158, 669, - /* 440 */ 806, 807, 1201, 1, 1045, 331, 1193, 806, 1047, 376, - /* 450 */ 1105, 333, 261, 375, 969, 1130, 1143, 1034, 671, 21, - /* 460 */ 319, 1107, 322, 190, 1144, 1147, 807, 854, 7, 410, - /* 470 */ 858, 149, 807, 1130, 7, 1105, 1158, 865, 377, 807, - /* 480 */ 57, 6, 1244, 331, 968, 147, 1104, 809, 812, 333, - /* 490 */ 1143, 1130, 53, 1130, 410, 116, 967, 374, 373, 1242, - /* 500 */ 410, 62, 1144, 1147, 1183, 820, 379, 410, 1182, 1179, - /* 510 */ 1158, 315, 809, 812, 30, 28, 332, 331, 809, 812, - /* 520 */ 966, 1130, 224, 333, 808, 809, 812, 1130, 896, 117, - /* 530 */ 965, 964, 91, 1130, 817, 62, 1144, 1147, 1183, 1133, - /* 540 */ 961, 806, 329, 1179, 960, 30, 28, 319, 945, 946, - /* 550 */ 959, 1143, 89, 224, 958, 808, 957, 1130, 131, 9, - /* 560 */ 8, 129, 155, 1190, 314, 413, 313, 1130, 1130, 1244, - /* 570 */ 884, 1158, 806, 807, 956, 7, 1130, 1130, 331, 178, - /* 580 */ 1043, 1130, 116, 87, 333, 955, 1242, 1130, 1130, 402, - /* 590 */ 1143, 1130, 177, 1130, 954, 323, 106, 1144, 1147, 330, - /* 600 */ 144, 410, 953, 126, 807, 326, 1, 110, 1197, 321, - /* 610 */ 1158, 1130, 371, 259, 963, 242, 125, 331, 59, 809, - /* 620 */ 812, 173, 1130, 333, 1026, 1192, 133, 1130, 1143, 132, - /* 630 */ 159, 1130, 410, 320, 1257, 62, 1144, 1147, 1183, 1130, - /* 640 */ 991, 135, 43, 1180, 134, 123, 137, 301, 1158, 136, - /* 650 */ 809, 812, 1143, 859, 343, 331, 986, 295, 152, 324, - /* 660 */ 141, 333, 276, 1143, 185, 1130, 984, 32, 223, 187, - /* 670 */ 844, 327, 1158, 196, 1144, 1147, 1099, 826, 278, 331, - /* 680 */ 122, 186, 104, 1158, 120, 333, 1143, 260, 281, 1130, - /* 690 */ 331, 32, 297, 1137, 118, 1213, 333, 196, 1144, 1147, - /* 700 */ 1130, 1143, 316, 798, 167, 1159, 1158, 1135, 195, 1144, - /* 710 */ 1147, 1033, 339, 331, 161, 172, 1143, 32, 165, 333, - /* 720 */ 2, 1158, 250, 1130, 238, 815, 93, 823, 331, 94, - /* 730 */ 119, 106, 1144, 1147, 333, 816, 1158, 1143, 1130, 311, - /* 740 */ 251, 221, 252, 331, 740, 735, 196, 1144, 1147, 333, - /* 750 */ 822, 41, 1143, 1130, 255, 124, 225, 1158, 96, 77, - /* 760 */ 821, 196, 1144, 1147, 331, 262, 264, 768, 1143, 1258, - /* 770 */ 333, 376, 1158, 772, 1130, 375, 778, 1050, 777, 331, - /* 780 */ 117, 351, 194, 1144, 1147, 333, 66, 94, 1158, 1130, - /* 790 */ 95, 1143, 96, 128, 97, 331, 1143, 197, 1144, 1147, - /* 800 */ 377, 333, 1046, 291, 130, 1130, 98, 1143, 94, 99, - /* 810 */ 1048, 1158, 1143, 188, 1144, 1147, 1158, 1044, 331, 374, - /* 820 */ 373, 140, 100, 331, 333, 101, 213, 1158, 1130, 333, - /* 830 */ 820, 1214, 1158, 1130, 331, 293, 198, 1144, 1147, 331, - /* 840 */ 333, 189, 1144, 1147, 1130, 333, 1224, 1143, 294, 1130, - /* 850 */ 1143, 302, 199, 1144, 1147, 337, 145, 1155, 1144, 1147, - /* 860 */ 812, 148, 299, 216, 298, 5, 1204, 1158, 1223, 312, - /* 870 */ 1158, 4, 884, 90, 331, 819, 151, 331, 109, 1198, - /* 880 */ 333, 33, 153, 333, 1130, 1143, 154, 1130, 218, 328, - /* 890 */ 17, 1241, 1154, 1144, 1147, 1153, 1144, 1147, 1259, 325, - /* 900 */ 1143, 160, 1165, 340, 1114, 1158, 335, 169, 336, 50, - /* 910 */ 341, 1113, 331, 226, 1061, 342, 1143, 179, 333, 52, - /* 920 */ 1158, 1143, 1130, 315, 181, 176, 409, 331, 191, 349, - /* 930 */ 204, 1144, 1147, 333, 183, 184, 1158, 1130, 1124, 192, - /* 940 */ 999, 1158, 1123, 331, 91, 203, 1144, 1147, 331, 333, - /* 950 */ 1122, 240, 1143, 1130, 333, 241, 1121, 208, 1130, 1039, - /* 960 */ 1038, 205, 1144, 1147, 89, 998, 202, 1144, 1147, 1000, - /* 970 */ 995, 983, 1158, 121, 114, 1190, 1191, 280, 1195, 331, - /* 980 */ 978, 1120, 267, 1111, 1037, 333, 266, 684, 997, 1130, - /* 990 */ 994, 256, 288, 258, 257, 982, 981, 193, 1144, 1147, - /* 1000 */ 209, 977, 207, 206, 1041, 265, 139, 65, 127, 781, - /* 1010 */ 283, 268, 1040, 783, 782, 277, 713, 712, 711, 138, - /* 1020 */ 992, 275, 210, 270, 710, 709, 274, 987, 708, 273, - /* 1030 */ 211, 271, 279, 985, 272, 212, 282, 976, 284, 975, - /* 1040 */ 286, 63, 1119, 1118, 1110, 38, 36, 142, 37, 44, - /* 1050 */ 3, 20, 32, 143, 14, 39, 146, 15, 306, 34, - /* 1060 */ 307, 31, 29, 27, 26, 25, 22, 909, 903, 47, - /* 1070 */ 11, 107, 150, 931, 930, 45, 31, 29, 27, 26, - /* 1080 */ 25, 902, 46, 8, 881, 880, 219, 935, 1109, 934, - /* 1090 */ 220, 993, 1135, 980, 19, 949, 936, 170, 827, 164, - /* 1100 */ 350, 156, 13, 35, 115, 18, 231, 354, 357, 360, - /* 1110 */ 907, 16, 845, 163, 166, 363, 53, 746, 168, 48, - /* 1120 */ 774, 704, 776, 775, 348, 49, 400, 1134, 769, 40, - /* 1130 */ 352, 399, 766, 355, 401, 378, 763, 757, 682, 358, - /* 1140 */ 171, 338, 174, 703, 361, 702, 755, 364, 701, 700, - /* 1150 */ 699, 698, 54, 55, 697, 56, 696, 705, 695, 694, - /* 1160 */ 761, 693, 692, 691, 690, 689, 403, 370, 688, 687, - /* 1170 */ 404, 979, 974, 407, 760, 408, 759, 810, 758, 182, - /* 1180 */ 411, 412, + /* 0 */ 1045, 1204, 225, 43, 24, 170, 362, 1200, 1206, 1095, + /* 10 */ 250, 269, 89, 31, 29, 27, 26, 25, 20, 1204, + /* 20 */ 1101, 27, 26, 25, 1084, 1200, 1205, 1106, 31, 29, + /* 30 */ 27, 26, 25, 361, 1000, 78, 209, 869, 77, 76, + /* 40 */ 75, 74, 73, 72, 71, 70, 69, 1091, 211, 414, + /* 50 */ 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, + /* 60 */ 403, 402, 401, 400, 399, 398, 397, 396, 1003, 105, + /* 70 */ 270, 1014, 881, 31, 29, 27, 26, 25, 1231, 132, + /* 80 */ 904, 349, 111, 249, 237, 346, 1216, 1180, 275, 78, + /* 90 */ 131, 22, 77, 76, 75, 74, 73, 72, 71, 70, + /* 100 */ 69, 31, 29, 27, 26, 25, 1231, 1317, 211, 291, + /* 110 */ 320, 286, 270, 346, 290, 44, 905, 289, 129, 287, + /* 120 */ 117, 345, 288, 348, 1315, 23, 232, 1192, 899, 900, + /* 130 */ 901, 902, 903, 907, 908, 909, 1082, 204, 1217, 1220, + /* 140 */ 904, 772, 385, 384, 383, 776, 382, 778, 779, 381, + /* 150 */ 781, 378, 109, 787, 375, 789, 790, 372, 369, 1151, + /* 160 */ 1216, 849, 128, 1144, 305, 224, 125, 43, 326, 189, + /* 170 */ 1149, 361, 1136, 30, 28, 941, 905, 847, 258, 242, + /* 180 */ 1231, 234, 395, 849, 1102, 23, 232, 346, 899, 900, + /* 190 */ 901, 902, 903, 907, 908, 909, 1204, 348, 361, 847, + /* 200 */ 896, 1192, 1200, 1205, 306, 362, 334, 848, 12, 118, + /* 210 */ 66, 61, 1217, 1220, 1256, 1216, 965, 279, 210, 1252, + /* 220 */ 1168, 10, 122, 121, 30, 28, 1106, 120, 1317, 848, + /* 230 */ 1317, 1, 234, 426, 849, 1231, 316, 963, 964, 966, + /* 240 */ 967, 117, 333, 117, 1216, 1315, 245, 1315, 10, 324, + /* 250 */ 847, 337, 348, 1171, 1173, 426, 1192, 349, 699, 12, + /* 260 */ 698, 850, 853, 1181, 1231, 362, 62, 1217, 1220, 1256, + /* 270 */ 66, 333, 283, 227, 1252, 112, 282, 285, 700, 330, + /* 280 */ 848, 348, 1, 850, 853, 1192, 1106, 166, 118, 9, + /* 290 */ 8, 59, 1216, 312, 1283, 62, 1217, 1220, 1256, 284, + /* 300 */ 92, 93, 227, 1252, 112, 1317, 426, 330, 1098, 106, + /* 310 */ 1073, 906, 1231, 31, 29, 27, 26, 25, 1316, 346, + /* 320 */ 21, 1216, 1315, 1284, 395, 1270, 90, 871, 92, 348, + /* 330 */ 910, 422, 421, 1192, 850, 853, 114, 1263, 1264, 870, + /* 340 */ 1268, 1231, 1267, 62, 1217, 1220, 1256, 334, 346, 118, + /* 350 */ 227, 1252, 1329, 698, 90, 1097, 1216, 867, 348, 917, + /* 360 */ 1231, 1290, 1192, 868, 163, 1263, 329, 346, 328, 277, + /* 370 */ 238, 1317, 62, 1217, 1220, 1256, 1231, 362, 104, 227, + /* 380 */ 1252, 1329, 1103, 346, 117, 1216, 1108, 388, 1315, 1025, + /* 390 */ 1313, 1093, 323, 348, 30, 28, 1192, 1192, 1106, 1024, + /* 400 */ 1270, 336, 234, 104, 849, 1231, 1083, 62, 1217, 1220, + /* 410 */ 1256, 1109, 346, 1151, 227, 1252, 1329, 1266, 362, 239, + /* 420 */ 847, 1151, 348, 359, 1149, 1274, 1192, 246, 165, 12, + /* 430 */ 1192, 334, 1149, 30, 28, 1015, 199, 1217, 1220, 1106, + /* 440 */ 1192, 234, 1216, 849, 194, 1023, 1022, 1021, 137, 196, + /* 450 */ 848, 135, 1, 1041, 319, 1317, 30, 28, 347, 847, + /* 460 */ 6, 195, 1231, 392, 234, 1216, 849, 391, 117, 346, + /* 470 */ 1020, 123, 1315, 1151, 118, 292, 426, 325, 321, 348, + /* 480 */ 1019, 1018, 847, 1192, 1172, 1231, 1192, 1192, 1192, 848, + /* 490 */ 393, 7, 346, 63, 1217, 1220, 1256, 1151, 872, 1089, + /* 500 */ 1255, 1252, 348, 1048, 850, 853, 1192, 948, 1150, 390, + /* 510 */ 389, 1192, 848, 869, 7, 426, 63, 1217, 1220, 1256, + /* 520 */ 244, 1192, 1192, 344, 1252, 30, 28, 303, 104, 30, + /* 530 */ 28, 64, 387, 234, 341, 849, 1108, 234, 426, 849, + /* 540 */ 301, 1216, 1270, 850, 853, 1002, 31, 29, 27, 26, + /* 550 */ 25, 847, 291, 936, 286, 847, 1017, 290, 118, 1265, + /* 560 */ 289, 1231, 287, 118, 1216, 288, 850, 853, 346, 87, + /* 570 */ 86, 85, 84, 83, 82, 81, 80, 79, 348, 867, + /* 580 */ 1036, 848, 1192, 7, 1231, 848, 251, 1, 1074, 263, + /* 590 */ 1016, 346, 107, 1217, 1220, 940, 429, 1192, 264, 151, + /* 600 */ 1216, 348, 294, 167, 362, 1192, 247, 426, 342, 360, + /* 610 */ 187, 426, 308, 88, 104, 63, 1217, 1220, 1256, 418, + /* 620 */ 1231, 186, 1108, 1253, 98, 1106, 1013, 346, 317, 335, + /* 630 */ 1330, 1192, 1034, 1012, 1011, 850, 853, 348, 362, 850, + /* 640 */ 853, 1192, 1145, 184, 233, 362, 60, 1010, 1009, 182, + /* 650 */ 248, 205, 1217, 1220, 297, 1008, 160, 1007, 1216, 1106, + /* 660 */ 1275, 936, 1232, 1216, 276, 262, 1106, 1192, 257, 256, + /* 670 */ 255, 254, 253, 338, 1192, 1192, 1006, 856, 1231, 139, + /* 680 */ 358, 1216, 138, 1231, 141, 346, 1216, 140, 1192, 1192, + /* 690 */ 346, 855, 994, 995, 311, 348, 1192, 147, 1192, 1192, + /* 700 */ 348, 1231, 313, 330, 1192, 331, 1231, 859, 346, 205, + /* 710 */ 1217, 1220, 1005, 346, 107, 1217, 1220, 1192, 348, 9, + /* 720 */ 8, 858, 1192, 348, 92, 231, 867, 1192, 1216, 939, + /* 730 */ 235, 1216, 205, 1217, 1220, 1286, 1216, 205, 1217, 1220, + /* 740 */ 31, 29, 27, 26, 25, 339, 330, 1170, 1231, 52, + /* 750 */ 90, 1231, 1331, 1192, 169, 346, 1231, 1081, 346, 332, + /* 760 */ 113, 1263, 1264, 346, 1268, 348, 1099, 92, 348, 1192, + /* 770 */ 2, 119, 1192, 348, 1216, 143, 260, 1192, 142, 203, + /* 780 */ 1217, 1220, 206, 1217, 1220, 962, 156, 197, 1217, 1220, + /* 790 */ 252, 259, 261, 90, 1231, 265, 1216, 41, 154, 881, + /* 800 */ 911, 346, 1216, 115, 1263, 1264, 875, 1268, 997, 998, + /* 810 */ 266, 348, 32, 267, 392, 1192, 1231, 124, 391, 874, + /* 820 */ 878, 58, 1231, 346, 127, 207, 1217, 1220, 1210, 346, + /* 830 */ 1216, 54, 32, 348, 842, 268, 1216, 1192, 271, 348, + /* 840 */ 1208, 393, 42, 1192, 278, 130, 32, 198, 1217, 1220, + /* 850 */ 1231, 873, 68, 208, 1217, 1220, 1231, 346, 175, 280, + /* 860 */ 390, 389, 223, 346, 1216, 1096, 354, 348, 181, 134, + /* 870 */ 173, 1192, 1092, 348, 765, 136, 100, 1192, 95, 101, + /* 880 */ 96, 1228, 1217, 1220, 1231, 1094, 98, 1227, 1217, 1220, + /* 890 */ 760, 346, 1216, 1090, 793, 797, 102, 803, 1216, 802, + /* 900 */ 310, 348, 41, 103, 146, 1192, 367, 96, 99, 97, + /* 910 */ 149, 98, 1231, 307, 309, 1226, 1217, 1220, 1231, 346, + /* 920 */ 96, 872, 318, 1297, 1287, 346, 352, 152, 853, 348, + /* 930 */ 315, 1296, 1216, 1192, 226, 348, 5, 159, 155, 1192, + /* 940 */ 1216, 322, 327, 214, 1217, 1220, 241, 240, 1277, 213, + /* 950 */ 1217, 1220, 1231, 314, 4, 161, 861, 936, 91, 346, + /* 960 */ 1231, 871, 110, 1271, 33, 162, 228, 346, 1216, 348, + /* 970 */ 1332, 340, 854, 1192, 296, 218, 343, 348, 17, 1238, + /* 980 */ 1179, 1192, 168, 215, 1217, 1220, 350, 1314, 1231, 304, + /* 990 */ 351, 212, 1217, 1220, 355, 346, 1178, 283, 356, 236, + /* 1000 */ 177, 282, 857, 145, 357, 348, 299, 179, 53, 1192, + /* 1010 */ 51, 293, 188, 219, 144, 217, 216, 190, 281, 202, + /* 1020 */ 1217, 1220, 1107, 365, 284, 185, 425, 200, 363, 201, + /* 1030 */ 192, 193, 1186, 825, 1163, 1162, 94, 1161, 1160, 40, + /* 1040 */ 1159, 1158, 39, 1157, 1156, 827, 1155, 1154, 1153, 1152, + /* 1050 */ 1047, 1185, 1176, 126, 1085, 711, 862, 853, 1046, 1044, + /* 1060 */ 272, 273, 274, 1033, 1032, 1029, 1087, 67, 133, 808, + /* 1070 */ 1086, 807, 1042, 1037, 740, 806, 1035, 739, 1028, 220, + /* 1080 */ 738, 1027, 221, 737, 222, 1184, 1183, 36, 1175, 150, + /* 1090 */ 300, 302, 3, 736, 735, 65, 45, 32, 14, 153, + /* 1100 */ 295, 37, 15, 158, 19, 298, 1208, 34, 11, 164, + /* 1110 */ 48, 8, 35, 16, 148, 897, 353, 1174, 180, 1001, + /* 1120 */ 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, + /* 1130 */ 983, 982, 229, 987, 986, 230, 863, 1001, 1001, 1001, + /* 1140 */ 1001, 1001, 1001, 178, 1001, 961, 1001, 108, 1001, 157, + /* 1150 */ 1001, 955, 46, 1001, 954, 933, 1001, 47, 1001, 1001, + /* 1160 */ 932, 1001, 988, 1001, 1001, 1001, 1001, 1001, 1001, 366, + /* 1170 */ 879, 13, 116, 18, 243, 172, 959, 174, 176, 1001, + /* 1180 */ 171, 49, 50, 1001, 1207, 38, 370, 1001, 373, 794, + /* 1190 */ 54, 376, 771, 1001, 379, 183, 1001, 364, 799, 1001, + /* 1200 */ 1001, 1001, 368, 731, 791, 371, 1043, 1001, 788, 723, + /* 1210 */ 1001, 428, 423, 801, 374, 786, 800, 782, 1001, 1001, + /* 1220 */ 1001, 730, 377, 729, 728, 709, 780, 727, 380, 394, + /* 1230 */ 785, 726, 1031, 725, 724, 417, 722, 721, 1030, 1026, + /* 1240 */ 416, 720, 55, 56, 57, 415, 420, 732, 719, 424, + /* 1250 */ 1001, 851, 718, 717, 716, 386, 715, 784, 191, 783, + /* 1260 */ 714, 419, 427, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 0, 169, 169, 212, 213, 196, 174, 174, 199, 184, - /* 10 */ 182, 202, 183, 12, 13, 14, 15, 16, 0, 190, - /* 20 */ 169, 193, 190, 190, 162, 174, 164, 12, 13, 14, - /* 30 */ 15, 16, 181, 192, 163, 163, 195, 196, 209, 21, - /* 40 */ 20, 190, 24, 25, 26, 27, 28, 29, 30, 31, - /* 50 */ 32, 50, 52, 53, 54, 55, 56, 57, 58, 59, - /* 60 */ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - /* 70 */ 70, 200, 200, 12, 13, 74, 12, 13, 14, 15, - /* 80 */ 16, 20, 20, 22, 21, 84, 171, 24, 25, 26, - /* 90 */ 27, 28, 29, 30, 31, 32, 52, 183, 54, 171, - /* 100 */ 39, 57, 227, 188, 60, 191, 62, 183, 180, 65, - /* 110 */ 169, 50, 0, 189, 50, 240, 188, 20, 194, 244, - /* 120 */ 119, 120, 20, 122, 123, 124, 125, 126, 127, 128, - /* 130 */ 129, 130, 71, 49, 73, 73, 24, 25, 26, 27, - /* 140 */ 28, 29, 30, 31, 32, 163, 205, 113, 84, 90, - /* 150 */ 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, - /* 160 */ 99, 102, 103, 104, 105, 106, 107, 21, 227, 187, - /* 170 */ 155, 156, 20, 139, 140, 73, 121, 163, 117, 118, - /* 180 */ 34, 240, 200, 119, 120, 244, 122, 123, 124, 125, - /* 190 */ 126, 127, 128, 129, 130, 134, 134, 183, 143, 144, - /* 200 */ 145, 146, 147, 163, 190, 12, 13, 14, 15, 16, - /* 210 */ 196, 169, 169, 4, 200, 163, 174, 174, 20, 205, - /* 220 */ 172, 173, 208, 209, 210, 211, 160, 187, 30, 215, - /* 230 */ 216, 134, 190, 190, 169, 183, 163, 74, 40, 174, - /* 240 */ 200, 227, 190, 45, 169, 47, 181, 20, 196, 49, - /* 250 */ 183, 88, 200, 206, 240, 190, 176, 190, 244, 179, - /* 260 */ 208, 209, 210, 211, 66, 190, 183, 215, 216, 217, - /* 270 */ 223, 205, 189, 200, 76, 77, 209, 194, 163, 166, - /* 280 */ 167, 229, 12, 13, 14, 210, 169, 235, 236, 20, - /* 290 */ 20, 174, 22, 227, 219, 220, 221, 222, 183, 224, - /* 300 */ 175, 169, 14, 15, 16, 190, 240, 190, 183, 39, - /* 310 */ 244, 196, 168, 163, 116, 200, 191, 168, 132, 133, - /* 320 */ 50, 20, 190, 208, 209, 210, 211, 178, 0, 185, - /* 330 */ 215, 216, 217, 183, 185, 12, 13, 14, 15, 16, - /* 340 */ 190, 71, 210, 73, 135, 60, 196, 175, 163, 64, - /* 350 */ 200, 236, 220, 221, 222, 183, 224, 74, 208, 209, - /* 360 */ 210, 211, 175, 191, 134, 215, 216, 217, 183, 99, - /* 370 */ 183, 88, 86, 163, 89, 190, 226, 49, 191, 163, - /* 380 */ 183, 196, 12, 13, 115, 200, 189, 117, 118, 0, - /* 390 */ 20, 194, 22, 208, 209, 210, 211, 74, 169, 183, - /* 400 */ 215, 216, 217, 174, 1, 2, 190, 12, 13, 39, - /* 410 */ 200, 226, 196, 12, 13, 20, 200, 22, 163, 190, - /* 420 */ 50, 20, 196, 22, 208, 209, 210, 211, 202, 206, - /* 430 */ 22, 215, 216, 217, 39, 22, 163, 20, 183, 22, - /* 440 */ 39, 71, 226, 73, 184, 190, 223, 39, 184, 60, - /* 450 */ 183, 196, 39, 64, 163, 200, 163, 0, 41, 119, - /* 460 */ 205, 194, 3, 208, 209, 210, 71, 127, 73, 99, - /* 470 */ 130, 74, 71, 200, 73, 183, 183, 74, 89, 71, - /* 480 */ 73, 44, 227, 190, 163, 88, 194, 117, 118, 196, - /* 490 */ 163, 200, 85, 200, 99, 240, 163, 108, 109, 244, - /* 500 */ 99, 208, 209, 210, 211, 20, 49, 99, 215, 216, - /* 510 */ 183, 169, 117, 118, 12, 13, 14, 190, 117, 118, - /* 520 */ 163, 200, 20, 196, 22, 117, 118, 200, 14, 134, - /* 530 */ 163, 163, 190, 200, 20, 208, 209, 210, 211, 163, - /* 540 */ 163, 39, 215, 216, 163, 12, 13, 205, 158, 159, - /* 550 */ 163, 163, 210, 20, 163, 22, 163, 200, 79, 1, - /* 560 */ 2, 82, 220, 221, 222, 19, 224, 200, 200, 227, - /* 570 */ 133, 183, 39, 71, 163, 73, 200, 200, 190, 33, - /* 580 */ 184, 200, 240, 37, 196, 163, 244, 200, 200, 43, - /* 590 */ 163, 200, 46, 200, 163, 88, 208, 209, 210, 50, - /* 600 */ 115, 99, 163, 33, 71, 88, 73, 37, 206, 150, - /* 610 */ 183, 200, 184, 43, 164, 169, 46, 190, 72, 117, - /* 620 */ 118, 75, 200, 196, 173, 223, 79, 200, 163, 82, - /* 630 */ 247, 200, 99, 245, 246, 208, 209, 210, 211, 200, - /* 640 */ 0, 79, 72, 216, 82, 75, 79, 238, 183, 82, - /* 650 */ 117, 118, 163, 74, 108, 190, 0, 111, 232, 152, - /* 660 */ 114, 196, 22, 163, 18, 200, 0, 88, 203, 23, - /* 670 */ 121, 154, 183, 208, 209, 210, 193, 74, 22, 190, - /* 680 */ 110, 35, 36, 183, 114, 196, 163, 166, 22, 200, - /* 690 */ 190, 88, 203, 73, 48, 207, 196, 208, 209, 210, - /* 700 */ 200, 163, 225, 74, 74, 183, 183, 87, 208, 209, - /* 710 */ 210, 0, 74, 190, 241, 74, 163, 88, 88, 196, - /* 720 */ 228, 183, 204, 200, 169, 20, 88, 20, 190, 88, - /* 730 */ 171, 208, 209, 210, 196, 20, 183, 163, 200, 239, - /* 740 */ 190, 203, 197, 190, 74, 74, 208, 209, 210, 196, - /* 750 */ 20, 171, 163, 200, 169, 171, 203, 183, 88, 88, - /* 760 */ 20, 208, 209, 210, 190, 165, 183, 74, 163, 246, - /* 770 */ 196, 60, 183, 74, 200, 64, 74, 183, 74, 190, - /* 780 */ 134, 88, 208, 209, 210, 196, 169, 88, 183, 200, - /* 790 */ 88, 163, 88, 183, 74, 190, 163, 208, 209, 210, - /* 800 */ 89, 196, 183, 204, 183, 200, 183, 163, 88, 183, - /* 810 */ 183, 183, 163, 208, 209, 210, 183, 183, 190, 108, - /* 820 */ 109, 168, 183, 190, 196, 183, 165, 183, 200, 196, - /* 830 */ 20, 207, 183, 200, 190, 190, 208, 209, 210, 190, - /* 840 */ 196, 208, 209, 210, 200, 196, 237, 163, 197, 200, - /* 850 */ 163, 142, 208, 209, 210, 141, 201, 208, 209, 210, - /* 860 */ 118, 201, 200, 200, 137, 149, 234, 183, 237, 148, - /* 870 */ 183, 136, 133, 190, 190, 20, 233, 190, 231, 206, - /* 880 */ 196, 131, 230, 196, 200, 163, 218, 200, 157, 153, - /* 890 */ 73, 243, 208, 209, 210, 208, 209, 210, 248, 151, - /* 900 */ 163, 242, 214, 112, 201, 183, 200, 190, 200, 168, - /* 910 */ 198, 201, 190, 200, 190, 197, 163, 179, 196, 73, - /* 920 */ 183, 163, 200, 169, 169, 168, 165, 190, 177, 186, - /* 930 */ 208, 209, 210, 196, 170, 161, 183, 200, 0, 177, - /* 940 */ 0, 183, 0, 190, 190, 208, 209, 210, 190, 196, - /* 950 */ 0, 66, 163, 200, 196, 87, 0, 35, 200, 0, - /* 960 */ 0, 208, 209, 210, 210, 0, 208, 209, 210, 0, - /* 970 */ 0, 0, 183, 44, 220, 221, 222, 4, 224, 190, - /* 980 */ 0, 0, 60, 0, 0, 196, 64, 51, 0, 200, - /* 990 */ 0, 39, 19, 44, 37, 0, 0, 208, 209, 210, - /* 1000 */ 78, 0, 80, 81, 0, 83, 33, 84, 82, 22, - /* 1010 */ 37, 89, 0, 39, 39, 42, 39, 39, 39, 46, - /* 1020 */ 0, 52, 22, 54, 39, 39, 57, 0, 39, 60, - /* 1030 */ 22, 62, 40, 0, 65, 22, 39, 0, 22, 0, - /* 1040 */ 22, 20, 0, 0, 0, 72, 115, 44, 75, 73, - /* 1050 */ 88, 2, 88, 110, 138, 88, 74, 138, 39, 132, - /* 1060 */ 88, 12, 13, 14, 15, 16, 2, 74, 74, 4, - /* 1070 */ 138, 73, 73, 39, 39, 73, 12, 13, 14, 15, - /* 1080 */ 16, 74, 73, 2, 74, 74, 39, 39, 0, 39, - /* 1090 */ 39, 0, 87, 0, 88, 249, 74, 44, 74, 74, - /* 1100 */ 39, 87, 73, 88, 87, 73, 39, 39, 39, 39, - /* 1110 */ 74, 88, 121, 87, 73, 39, 85, 22, 73, 73, - /* 1120 */ 22, 22, 39, 39, 86, 73, 37, 87, 74, 73, - /* 1130 */ 73, 39, 74, 73, 44, 50, 74, 74, 51, 73, - /* 1140 */ 110, 113, 87, 39, 73, 39, 74, 73, 39, 39, - /* 1150 */ 39, 39, 73, 73, 39, 73, 22, 71, 39, 39, - /* 1160 */ 101, 39, 39, 39, 39, 39, 39, 89, 39, 39, - /* 1170 */ 38, 0, 0, 22, 101, 21, 101, 22, 101, 22, - /* 1180 */ 21, 20, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1190 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1200 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1210 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1220 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1230 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1240 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1250 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1260 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1270 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1280 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1290 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1300 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1310 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1320 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1330 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 1340 */ 249, 249, + /* 0 */ 0, 207, 190, 174, 221, 222, 172, 213, 214, 187, + /* 10 */ 172, 177, 183, 12, 13, 14, 15, 16, 2, 207, + /* 20 */ 191, 14, 15, 16, 0, 213, 214, 193, 12, 13, + /* 30 */ 14, 15, 16, 20, 163, 21, 198, 20, 24, 25, + /* 40 */ 26, 27, 28, 29, 30, 31, 32, 187, 47, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + /* 60 */ 60, 61, 62, 63, 64, 65, 66, 67, 0, 165, + /* 70 */ 46, 167, 71, 12, 13, 14, 15, 16, 186, 33, + /* 80 */ 79, 203, 36, 212, 206, 193, 166, 209, 42, 21, + /* 90 */ 44, 2, 24, 25, 26, 27, 28, 29, 30, 31, + /* 100 */ 32, 12, 13, 14, 15, 16, 186, 236, 47, 49, + /* 110 */ 218, 51, 46, 193, 54, 69, 115, 57, 72, 59, + /* 120 */ 249, 47, 62, 203, 253, 124, 125, 207, 127, 128, + /* 130 */ 129, 130, 131, 132, 133, 134, 0, 217, 218, 219, + /* 140 */ 79, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 150 */ 94, 95, 185, 97, 98, 99, 100, 101, 102, 186, + /* 160 */ 166, 22, 116, 196, 172, 192, 120, 174, 248, 179, + /* 170 */ 197, 20, 182, 12, 13, 14, 115, 38, 63, 190, + /* 180 */ 186, 20, 46, 22, 191, 124, 125, 193, 127, 128, + /* 190 */ 129, 130, 131, 132, 133, 134, 207, 203, 20, 38, + /* 200 */ 126, 207, 213, 214, 212, 172, 212, 68, 47, 137, + /* 210 */ 177, 217, 218, 219, 220, 166, 126, 184, 224, 225, + /* 220 */ 193, 70, 107, 108, 12, 13, 193, 200, 236, 68, + /* 230 */ 236, 70, 20, 94, 22, 186, 146, 147, 148, 149, + /* 240 */ 150, 249, 193, 249, 166, 253, 195, 253, 70, 20, + /* 250 */ 38, 3, 203, 202, 203, 94, 207, 203, 20, 47, + /* 260 */ 22, 122, 123, 209, 186, 172, 217, 218, 219, 220, + /* 270 */ 177, 193, 57, 224, 225, 226, 61, 184, 40, 172, + /* 280 */ 68, 203, 70, 122, 123, 207, 193, 238, 137, 1, + /* 290 */ 2, 171, 166, 244, 245, 217, 218, 219, 220, 84, + /* 300 */ 193, 181, 224, 225, 226, 236, 94, 172, 188, 175, + /* 310 */ 176, 115, 186, 12, 13, 14, 15, 16, 249, 193, + /* 320 */ 124, 166, 253, 245, 46, 215, 219, 20, 193, 203, + /* 330 */ 134, 169, 170, 207, 122, 123, 229, 230, 231, 20, + /* 340 */ 233, 186, 232, 217, 218, 219, 220, 212, 193, 137, + /* 350 */ 224, 225, 226, 22, 219, 166, 166, 20, 203, 71, + /* 360 */ 186, 235, 207, 20, 229, 230, 231, 193, 233, 38, + /* 370 */ 178, 236, 217, 218, 219, 220, 186, 172, 186, 224, + /* 380 */ 225, 226, 177, 193, 249, 166, 194, 81, 253, 166, + /* 390 */ 235, 187, 218, 203, 12, 13, 207, 207, 193, 166, + /* 400 */ 215, 153, 20, 186, 22, 186, 0, 217, 218, 219, + /* 410 */ 220, 194, 193, 186, 224, 225, 226, 232, 172, 192, + /* 420 */ 38, 186, 203, 177, 197, 235, 207, 192, 121, 47, + /* 430 */ 207, 212, 197, 12, 13, 167, 217, 218, 219, 193, + /* 440 */ 207, 20, 166, 22, 18, 166, 166, 166, 74, 23, + /* 450 */ 68, 77, 70, 0, 119, 236, 12, 13, 14, 38, + /* 460 */ 43, 35, 186, 57, 20, 166, 22, 61, 249, 193, + /* 470 */ 166, 45, 253, 186, 137, 22, 94, 142, 143, 203, + /* 480 */ 166, 166, 38, 207, 197, 186, 207, 207, 207, 68, + /* 490 */ 84, 70, 193, 217, 218, 219, 220, 186, 20, 187, + /* 500 */ 224, 225, 203, 0, 122, 123, 207, 14, 197, 103, + /* 510 */ 104, 207, 68, 20, 70, 94, 217, 218, 219, 220, + /* 520 */ 178, 207, 207, 224, 225, 12, 13, 21, 186, 12, + /* 530 */ 13, 105, 187, 20, 83, 22, 194, 20, 94, 22, + /* 540 */ 34, 166, 215, 122, 123, 0, 12, 13, 14, 15, + /* 550 */ 16, 38, 49, 136, 51, 38, 166, 54, 137, 232, + /* 560 */ 57, 186, 59, 137, 166, 62, 122, 123, 193, 24, + /* 570 */ 25, 26, 27, 28, 29, 30, 31, 32, 203, 20, + /* 580 */ 0, 68, 207, 70, 186, 68, 27, 70, 176, 30, + /* 590 */ 166, 193, 217, 218, 219, 4, 19, 207, 39, 121, + /* 600 */ 166, 203, 22, 256, 172, 207, 178, 94, 157, 177, + /* 610 */ 33, 94, 71, 36, 186, 217, 218, 219, 220, 42, + /* 620 */ 186, 44, 194, 225, 83, 193, 166, 193, 247, 254, + /* 630 */ 255, 207, 0, 166, 166, 122, 123, 203, 172, 122, + /* 640 */ 123, 207, 196, 177, 210, 172, 69, 166, 166, 72, + /* 650 */ 177, 217, 218, 219, 22, 166, 241, 166, 166, 193, + /* 660 */ 135, 136, 186, 166, 169, 106, 193, 207, 109, 110, + /* 670 */ 111, 112, 113, 83, 207, 207, 166, 38, 186, 74, + /* 680 */ 103, 166, 77, 186, 74, 193, 166, 77, 207, 207, + /* 690 */ 193, 38, 158, 159, 117, 203, 207, 120, 207, 207, + /* 700 */ 203, 186, 210, 172, 207, 234, 186, 68, 193, 217, + /* 710 */ 218, 219, 166, 193, 217, 218, 219, 207, 203, 1, + /* 720 */ 2, 68, 207, 203, 193, 210, 20, 207, 166, 138, + /* 730 */ 210, 166, 217, 218, 219, 216, 166, 217, 218, 219, + /* 740 */ 12, 13, 14, 15, 16, 155, 172, 172, 186, 171, + /* 750 */ 219, 186, 255, 207, 250, 193, 186, 0, 193, 228, + /* 760 */ 229, 230, 231, 193, 233, 203, 188, 193, 203, 207, + /* 770 */ 237, 114, 207, 203, 166, 74, 115, 207, 77, 217, + /* 780 */ 218, 219, 217, 218, 219, 71, 71, 217, 218, 219, + /* 790 */ 201, 199, 199, 219, 186, 172, 166, 83, 83, 71, + /* 800 */ 71, 193, 166, 229, 230, 231, 20, 233, 161, 162, + /* 810 */ 211, 203, 83, 193, 57, 207, 186, 174, 61, 20, + /* 820 */ 71, 70, 186, 193, 174, 217, 218, 219, 70, 193, + /* 830 */ 166, 80, 83, 203, 71, 204, 166, 207, 172, 203, + /* 840 */ 82, 84, 174, 207, 168, 174, 83, 217, 218, 219, + /* 850 */ 186, 20, 172, 217, 218, 219, 186, 193, 71, 186, + /* 860 */ 103, 104, 168, 193, 166, 186, 71, 203, 71, 186, + /* 870 */ 83, 207, 186, 203, 71, 186, 186, 207, 83, 186, + /* 880 */ 83, 217, 218, 219, 186, 186, 83, 217, 218, 219, + /* 890 */ 71, 193, 166, 186, 71, 71, 186, 71, 166, 71, + /* 900 */ 204, 203, 83, 186, 171, 207, 83, 83, 71, 83, + /* 910 */ 171, 83, 186, 211, 193, 217, 218, 219, 186, 193, + /* 920 */ 83, 20, 145, 246, 216, 193, 144, 208, 123, 203, + /* 930 */ 207, 246, 166, 207, 207, 203, 152, 242, 208, 207, + /* 940 */ 166, 207, 151, 217, 218, 219, 12, 13, 243, 217, + /* 950 */ 218, 219, 186, 140, 139, 239, 22, 136, 193, 193, + /* 960 */ 186, 20, 240, 215, 114, 227, 160, 193, 166, 203, + /* 970 */ 257, 154, 38, 207, 4, 35, 156, 203, 70, 223, + /* 980 */ 208, 207, 251, 217, 218, 219, 207, 252, 186, 19, + /* 990 */ 207, 217, 218, 219, 118, 193, 208, 57, 205, 207, + /* 1000 */ 193, 61, 68, 33, 204, 203, 36, 171, 70, 207, + /* 1010 */ 171, 41, 182, 73, 44, 75, 76, 172, 78, 217, + /* 1020 */ 218, 219, 193, 189, 84, 171, 168, 180, 94, 180, + /* 1030 */ 173, 164, 0, 82, 0, 0, 114, 0, 0, 69, + /* 1040 */ 0, 0, 72, 0, 0, 22, 0, 0, 0, 0, + /* 1050 */ 0, 0, 0, 43, 0, 48, 122, 123, 0, 0, + /* 1060 */ 38, 36, 43, 0, 0, 0, 0, 79, 77, 38, + /* 1070 */ 0, 38, 0, 0, 38, 22, 0, 38, 0, 22, + /* 1080 */ 38, 0, 22, 38, 22, 0, 0, 121, 0, 116, + /* 1090 */ 22, 22, 83, 38, 38, 20, 70, 83, 141, 71, + /* 1100 */ 39, 83, 141, 83, 83, 38, 82, 135, 141, 82, + /* 1110 */ 4, 2, 83, 83, 43, 126, 119, 0, 116, 258, + /* 1120 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1130 */ 38, 38, 38, 38, 38, 38, 22, 258, 258, 258, + /* 1140 */ 258, 258, 258, 43, 258, 71, 258, 70, 258, 70, + /* 1150 */ 258, 71, 70, 258, 71, 71, 258, 70, 258, 258, + /* 1160 */ 71, 258, 71, 258, 258, 258, 258, 258, 258, 38, + /* 1170 */ 71, 70, 82, 70, 38, 71, 71, 70, 70, 258, + /* 1180 */ 82, 70, 70, 258, 82, 70, 38, 258, 38, 71, + /* 1190 */ 80, 38, 22, 258, 38, 82, 258, 81, 22, 258, + /* 1200 */ 258, 258, 70, 22, 71, 70, 0, 258, 71, 22, + /* 1210 */ 258, 20, 22, 38, 70, 96, 38, 71, 258, 258, + /* 1220 */ 258, 38, 70, 38, 38, 48, 71, 38, 70, 47, + /* 1230 */ 96, 38, 0, 38, 38, 43, 38, 38, 0, 0, + /* 1240 */ 36, 38, 70, 70, 70, 38, 37, 68, 38, 21, + /* 1250 */ 258, 22, 38, 38, 38, 84, 38, 96, 22, 96, + /* 1260 */ 38, 38, 21, 258, 258, 258, 258, 258, 258, 258, + /* 1270 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1280 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1290 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1300 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1310 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1320 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1330 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1340 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1350 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1360 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1370 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1380 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1390 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1400 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1410 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1420 */ 258, 258, 258, 258, 258, 258, }; -#define YY_SHIFT_COUNT (413) +#define YY_SHIFT_COUNT (429) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1172) +#define YY_SHIFT_MAX (1241) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 646, 61, 270, 370, 370, 370, 370, 395, 370, 370, - /* 10 */ 62, 401, 533, 502, 401, 401, 401, 401, 401, 401, - /* 20 */ 401, 401, 401, 401, 401, 401, 401, 401, 401, 401, - /* 30 */ 401, 401, 401, 102, 102, 102, 97, 20, 20, 408, - /* 40 */ 408, 20, 20, 84, 152, 227, 227, 230, 301, 152, - /* 50 */ 20, 20, 152, 20, 152, 301, 152, 152, 20, 200, - /* 60 */ 1, 64, 64, 63, 922, 408, 44, 408, 408, 408, - /* 70 */ 408, 408, 408, 408, 408, 408, 408, 408, 408, 408, - /* 80 */ 408, 408, 408, 408, 408, 408, 408, 417, 328, 269, - /* 90 */ 269, 269, 457, 301, 152, 152, 152, 286, 59, 59, - /* 100 */ 59, 59, 59, 18, 198, 969, 15, 55, 285, 34, - /* 110 */ 413, 485, 186, 437, 186, 514, 459, 209, 705, 707, - /* 120 */ 84, 715, 730, 84, 705, 84, 740, 152, 152, 152, - /* 130 */ 152, 152, 152, 152, 152, 152, 152, 152, 705, 740, - /* 140 */ 707, 200, 715, 730, 810, 709, 714, 742, 709, 714, - /* 150 */ 742, 716, 721, 727, 735, 739, 715, 855, 750, 731, - /* 160 */ 736, 748, 817, 152, 714, 742, 742, 714, 742, 791, - /* 170 */ 715, 730, 286, 200, 715, 846, 705, 200, 740, 1182, - /* 180 */ 1182, 1182, 1182, 0, 112, 546, 570, 973, 1049, 1064, - /* 190 */ 323, 389, 711, 193, 193, 193, 193, 193, 193, 193, - /* 200 */ 403, 340, 288, 288, 288, 288, 479, 547, 562, 567, - /* 210 */ 640, 656, 666, 146, 163, 283, 397, 558, 390, 507, - /* 220 */ 517, 579, 549, 603, 620, 629, 630, 638, 641, 670, - /* 230 */ 671, 693, 699, 702, 704, 720, 407, 938, 940, 942, - /* 240 */ 950, 885, 868, 956, 959, 960, 965, 970, 971, 980, - /* 250 */ 981, 983, 929, 984, 936, 988, 990, 952, 957, 949, - /* 260 */ 995, 996, 1001, 1004, 923, 926, 974, 975, 987, 1012, - /* 270 */ 977, 978, 979, 985, 986, 989, 1020, 1000, 1027, 1008, - /* 280 */ 992, 1033, 1013, 997, 1037, 1016, 1039, 1018, 1021, 1042, - /* 290 */ 1043, 931, 1044, 976, 1003, 943, 962, 964, 916, 982, - /* 300 */ 967, 993, 998, 999, 994, 1002, 1007, 1019, 972, 1005, - /* 310 */ 1009, 1006, 919, 1010, 1011, 1014, 927, 1015, 1017, 1022, - /* 320 */ 1023, 932, 1065, 1034, 1035, 1047, 1048, 1050, 1051, 1081, - /* 330 */ 991, 1026, 1024, 1029, 1032, 1025, 1036, 1041, 1045, 1028, - /* 340 */ 1046, 1088, 1053, 1030, 1052, 1031, 1040, 1055, 1056, 1038, - /* 350 */ 1054, 1061, 1067, 1057, 1058, 1068, 1060, 1062, 1069, 1066, - /* 360 */ 1063, 1070, 1071, 1072, 1076, 1074, 1059, 1073, 1075, 1077, - /* 370 */ 1095, 1078, 1079, 1080, 1082, 1083, 1084, 1098, 1087, 1085, - /* 380 */ 1086, 1099, 1104, 1106, 1109, 1110, 1111, 1112, 1115, 1134, - /* 390 */ 1119, 1120, 1122, 1123, 1124, 1125, 1126, 1129, 1130, 1091, - /* 400 */ 1092, 1089, 1090, 1093, 1127, 1132, 1171, 1172, 1151, 1154, - /* 410 */ 1155, 1157, 1159, 1161, + /* 0 */ 426, 212, 161, 382, 382, 382, 382, 421, 382, 382, + /* 10 */ 151, 513, 517, 444, 513, 513, 513, 513, 513, 513, + /* 20 */ 513, 513, 513, 513, 513, 513, 513, 513, 513, 513, + /* 30 */ 513, 513, 513, 178, 178, 178, 337, 934, 934, 13, + /* 40 */ 13, 934, 13, 13, 66, 17, 229, 229, 72, 319, + /* 50 */ 17, 13, 13, 17, 13, 17, 319, 17, 17, 13, + /* 60 */ 278, 1, 61, 61, 559, 14, 940, 139, 60, 139, + /* 70 */ 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, + /* 80 */ 139, 139, 139, 139, 139, 139, 139, 139, 238, 24, + /* 90 */ 307, 307, 307, 136, 343, 319, 17, 17, 17, 306, + /* 100 */ 56, 56, 56, 56, 56, 68, 503, 534, 90, 215, + /* 110 */ 335, 331, 478, 525, 417, 525, 493, 248, 591, 706, + /* 120 */ 657, 661, 661, 706, 786, 66, 343, 799, 66, 66, + /* 130 */ 706, 66, 831, 17, 17, 17, 17, 17, 17, 17, + /* 140 */ 17, 17, 17, 17, 706, 831, 786, 278, 343, 799, + /* 150 */ 278, 901, 777, 782, 805, 777, 782, 805, 805, 784, + /* 160 */ 791, 813, 815, 821, 343, 941, 850, 806, 820, 817, + /* 170 */ 908, 17, 782, 805, 805, 782, 805, 876, 343, 799, + /* 180 */ 278, 306, 278, 343, 938, 706, 278, 831, 1263, 1263, + /* 190 */ 1263, 1263, 0, 545, 577, 46, 970, 16, 89, 728, + /* 200 */ 406, 757, 301, 301, 301, 301, 301, 301, 301, 115, + /* 210 */ 288, 196, 7, 7, 7, 7, 374, 605, 610, 701, + /* 220 */ 453, 580, 632, 506, 541, 714, 715, 718, 647, 590, + /* 230 */ 451, 729, 74, 749, 758, 763, 787, 795, 797, 803, + /* 240 */ 639, 653, 819, 823, 824, 826, 828, 837, 751, 1032, + /* 250 */ 951, 1034, 1035, 922, 1037, 1038, 1040, 1041, 1043, 1044, + /* 260 */ 1023, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1010, 1054, + /* 270 */ 1007, 1058, 1059, 1022, 1025, 1019, 1063, 1064, 1065, 1066, + /* 280 */ 988, 991, 1031, 1033, 1053, 1070, 1036, 1039, 1042, 1045, + /* 290 */ 1055, 1056, 1072, 1057, 1073, 1060, 1061, 1076, 1062, 1067, + /* 300 */ 1078, 1068, 1081, 1069, 1075, 1085, 1086, 966, 1088, 1026, + /* 310 */ 1071, 973, 1009, 1014, 957, 1028, 1018, 1074, 1077, 1079, + /* 320 */ 1080, 1082, 1083, 1020, 1024, 1087, 1021, 961, 1084, 1089, + /* 330 */ 1027, 972, 1029, 1090, 1091, 1030, 967, 1106, 1092, 1093, + /* 340 */ 1094, 1095, 1096, 1097, 1109, 989, 1098, 1099, 1101, 1103, + /* 350 */ 1104, 1105, 1107, 1108, 997, 1111, 1117, 1100, 1002, 1112, + /* 360 */ 1110, 1102, 1113, 1114, 1115, 1116, 1118, 1131, 1136, 1132, + /* 370 */ 1133, 1148, 1135, 1137, 1150, 1144, 1146, 1153, 1152, 1155, + /* 380 */ 1156, 1158, 1119, 1134, 1161, 1163, 1170, 1171, 1172, 1173, + /* 390 */ 1174, 1175, 1178, 1176, 1177, 1182, 1179, 1181, 1183, 1185, + /* 400 */ 1186, 1189, 1193, 1195, 1196, 1187, 1198, 1199, 1203, 1210, + /* 410 */ 1214, 1215, 1216, 1218, 1222, 1206, 1207, 1204, 1192, 1232, + /* 420 */ 1223, 1209, 1238, 1239, 1190, 1228, 1229, 1236, 1241, 1191, }; -#define YY_REDUCE_COUNT (182) -#define YY_REDUCE_MIN (-209) -#define YY_REDUCE_MAX (789) +#define YY_REDUCE_COUNT (191) +#define YY_REDUCE_MIN (-217) +#define YY_REDUCE_MAX (867) static const short yy_reduce_ofst[] = { - /* 0 */ 66, 14, 52, 115, 150, 185, 216, 255, 293, 327, - /* 10 */ 342, 388, 427, 465, 489, 500, 523, 538, 553, 574, - /* 20 */ 589, 605, 628, 633, 644, 649, 684, 687, 722, 737, - /* 30 */ 753, 758, 789, 75, 132, 754, -59, -149, 65, -18, - /* 40 */ 40, -168, -167, -72, -76, -171, 67, -125, -191, 125, - /* 50 */ 42, 43, 83, 117, 172, -159, 197, 187, 229, 149, - /* 60 */ -209, -209, -209, -138, -172, -129, 48, -128, 73, 210, - /* 70 */ 273, 291, 321, 333, 357, 367, 368, 376, 377, 381, - /* 80 */ 387, 391, 393, 411, 422, 431, 439, 113, -85, 47, - /* 90 */ 223, 402, 144, 226, -86, 267, 292, 80, -175, 260, - /* 100 */ 264, 396, 428, 450, 446, 451, 383, 409, 483, 426, - /* 110 */ 521, 488, 477, 477, 477, 522, 473, 492, 555, 518, - /* 120 */ 559, 550, 545, 580, 585, 584, 600, 583, 594, 610, - /* 130 */ 619, 621, 623, 626, 627, 634, 639, 642, 617, 661, - /* 140 */ 599, 653, 645, 651, 624, 609, 655, 662, 631, 660, - /* 150 */ 663, 632, 643, 647, 652, 477, 683, 673, 668, 650, - /* 160 */ 648, 659, 688, 522, 703, 706, 708, 710, 713, 712, - /* 170 */ 717, 718, 738, 741, 724, 743, 755, 757, 761, 751, - /* 180 */ 762, 764, 774, + /* 0 */ -129, -6, 49, 78, 126, 155, 190, 219, 276, 299, + /* 10 */ 135, 375, 398, 434, 492, -80, 497, 515, 520, 562, + /* 20 */ 565, 570, 608, 630, 636, 664, 670, 698, 726, 732, + /* 30 */ 766, 774, 802, 531, 107, 574, -8, -188, -11, 33, + /* 40 */ 93, -206, -166, 205, -171, -27, -108, 174, 69, -122, + /* 50 */ 192, 246, 432, 227, 466, 342, 51, 235, 428, 473, + /* 60 */ 120, -217, -217, -217, -162, -96, -33, 189, 134, 223, + /* 70 */ 233, 279, 280, 281, 304, 314, 315, 390, 424, 460, + /* 80 */ 467, 468, 481, 482, 489, 491, 510, 546, 162, -7, + /* 90 */ 110, 185, 327, 578, 27, 54, 217, 287, 311, -10, + /* 100 */ -178, -140, 204, 312, 345, 268, 412, 347, 381, 446, + /* 110 */ 415, 495, 519, 471, 471, 471, 476, 504, 533, 575, + /* 120 */ 589, 592, 593, 623, 599, 643, 620, 631, 650, 668, + /* 130 */ 666, 671, 676, 673, 679, 683, 686, 689, 690, 693, + /* 140 */ 699, 707, 710, 717, 680, 694, 702, 733, 721, 696, + /* 150 */ 739, 708, 677, 719, 723, 685, 730, 727, 734, 705, + /* 160 */ 695, 722, 716, 471, 765, 748, 738, 713, 735, 731, + /* 170 */ 756, 476, 772, 779, 783, 788, 792, 793, 807, 800, + /* 180 */ 836, 830, 839, 829, 834, 845, 854, 858, 847, 849, + /* 190 */ 857, 867, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 10 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 20 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 30 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 40 */ 947, 947, 947, 1004, 947, 947, 947, 947, 947, 947, - /* 50 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 1002, - /* 60 */ 947, 1185, 947, 947, 947, 947, 947, 947, 947, 947, - /* 70 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 80 */ 947, 947, 947, 947, 947, 947, 947, 947, 1004, 1196, - /* 90 */ 1196, 1196, 1002, 947, 947, 947, 947, 1089, 947, 947, - /* 100 */ 947, 947, 947, 947, 947, 947, 1260, 947, 1042, 1220, - /* 110 */ 947, 1212, 1188, 1202, 1189, 947, 1245, 1205, 947, 947, - /* 120 */ 1004, 947, 947, 1004, 947, 1004, 947, 947, 947, 947, - /* 130 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 140 */ 947, 1002, 947, 947, 947, 1227, 1225, 947, 1227, 1225, - /* 150 */ 947, 1239, 1235, 1218, 1216, 1202, 947, 947, 947, 1263, - /* 160 */ 1251, 1247, 947, 947, 1225, 947, 947, 1225, 947, 1112, - /* 170 */ 947, 947, 947, 1002, 947, 1058, 947, 1002, 947, 1092, - /* 180 */ 1092, 1005, 952, 947, 947, 947, 947, 947, 947, 947, - /* 190 */ 947, 947, 947, 1157, 1238, 1237, 1156, 1162, 1161, 1160, - /* 200 */ 947, 947, 1151, 1152, 1150, 1149, 947, 947, 947, 947, - /* 210 */ 947, 947, 947, 947, 947, 947, 947, 1186, 947, 1248, - /* 220 */ 1252, 947, 947, 947, 1136, 947, 947, 947, 947, 947, - /* 230 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 240 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 250 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 260 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 270 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 280 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 290 */ 947, 947, 947, 947, 947, 947, 1209, 1219, 947, 947, - /* 300 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 1136, - /* 310 */ 947, 1236, 947, 1195, 1191, 947, 947, 1187, 947, 947, - /* 320 */ 1246, 947, 947, 947, 947, 947, 947, 947, 947, 1181, - /* 330 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 340 */ 947, 947, 947, 947, 947, 947, 1135, 947, 947, 947, - /* 350 */ 947, 947, 947, 1086, 947, 947, 947, 947, 947, 947, - /* 360 */ 947, 947, 947, 947, 947, 947, 1071, 1069, 1068, 1067, - /* 370 */ 947, 1064, 947, 947, 947, 947, 947, 947, 947, 947, - /* 380 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 390 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 400 */ 947, 947, 947, 947, 947, 947, 947, 947, 947, 947, - /* 410 */ 947, 947, 947, 947, + /* 0 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 10 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 20 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 30 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 40 */ 999, 999, 999, 999, 1052, 999, 999, 999, 999, 999, + /* 50 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 60 */ 1050, 999, 1258, 999, 1164, 999, 999, 999, 999, 999, + /* 70 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 80 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 1052, + /* 90 */ 1269, 1269, 1269, 1050, 999, 999, 999, 999, 999, 1135, + /* 100 */ 999, 999, 999, 999, 999, 999, 999, 1333, 999, 1088, + /* 110 */ 1293, 999, 1285, 1261, 1275, 1262, 999, 1318, 1278, 999, + /* 120 */ 1169, 1166, 1166, 999, 999, 1052, 999, 999, 1052, 1052, + /* 130 */ 999, 1052, 999, 999, 999, 999, 999, 999, 999, 999, + /* 140 */ 999, 999, 999, 999, 999, 999, 999, 1050, 999, 999, + /* 150 */ 1050, 999, 1300, 1298, 999, 1300, 1298, 999, 999, 1312, + /* 160 */ 1308, 1291, 1289, 1275, 999, 999, 999, 1336, 1324, 1320, + /* 170 */ 999, 999, 1298, 999, 999, 1298, 999, 1177, 999, 999, + /* 180 */ 1050, 999, 1050, 999, 1104, 999, 1050, 999, 1138, 1138, + /* 190 */ 1053, 1004, 999, 999, 999, 999, 999, 999, 999, 999, + /* 200 */ 999, 999, 1230, 1311, 1310, 1229, 1235, 1234, 1233, 999, + /* 210 */ 999, 999, 1224, 1225, 1223, 1222, 999, 999, 999, 999, + /* 220 */ 999, 999, 999, 999, 999, 999, 999, 1259, 999, 1321, + /* 230 */ 1325, 999, 999, 999, 1209, 999, 999, 999, 999, 999, + /* 240 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 250 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 260 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 270 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 280 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 290 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 300 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 310 */ 999, 999, 1282, 1292, 999, 999, 999, 999, 999, 999, + /* 320 */ 999, 999, 999, 999, 1209, 999, 1309, 999, 1268, 1264, + /* 330 */ 999, 999, 1260, 999, 999, 1319, 999, 999, 999, 999, + /* 340 */ 999, 999, 999, 999, 1254, 999, 999, 999, 999, 999, + /* 350 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 360 */ 999, 1208, 999, 999, 999, 999, 999, 999, 999, 1132, + /* 370 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 380 */ 999, 999, 1117, 1115, 1114, 1113, 999, 1110, 999, 999, + /* 390 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 400 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 410 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 420 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, }; /********** End of lemon-generated parsing tables *****************************/ @@ -723,219 +742,228 @@ static const char *const yyTokenName[] = { /* 33 */ "USER", /* 34 */ "PRIVILEGE", /* 35 */ "DROP", - /* 36 */ "SHOW", - /* 37 */ "DNODE", - /* 38 */ "PORT", - /* 39 */ "NK_INTEGER", - /* 40 */ "DNODES", - /* 41 */ "NK_IPTOKEN", - /* 42 */ "LOCAL", - /* 43 */ "QNODE", - /* 44 */ "ON", - /* 45 */ "QNODES", - /* 46 */ "DATABASE", - /* 47 */ "DATABASES", - /* 48 */ "USE", - /* 49 */ "IF", - /* 50 */ "NOT", - /* 51 */ "EXISTS", - /* 52 */ "BLOCKS", - /* 53 */ "CACHE", - /* 54 */ "CACHELAST", - /* 55 */ "COMP", - /* 56 */ "DAYS", - /* 57 */ "FSYNC", - /* 58 */ "MAXROWS", - /* 59 */ "MINROWS", - /* 60 */ "KEEP", - /* 61 */ "PRECISION", - /* 62 */ "QUORUM", - /* 63 */ "REPLICA", - /* 64 */ "TTL", - /* 65 */ "WAL", - /* 66 */ "VGROUPS", - /* 67 */ "SINGLE_STABLE", - /* 68 */ "STREAM_MODE", - /* 69 */ "RETENTIONS", - /* 70 */ "FILE_FACTOR", - /* 71 */ "NK_FLOAT", - /* 72 */ "TABLE", - /* 73 */ "NK_LP", - /* 74 */ "NK_RP", - /* 75 */ "STABLE", - /* 76 */ "TABLES", - /* 77 */ "STABLES", - /* 78 */ "ADD", - /* 79 */ "COLUMN", - /* 80 */ "MODIFY", - /* 81 */ "RENAME", - /* 82 */ "TAG", - /* 83 */ "SET", - /* 84 */ "NK_EQ", - /* 85 */ "USING", - /* 86 */ "TAGS", - /* 87 */ "NK_DOT", - /* 88 */ "NK_COMMA", - /* 89 */ "COMMENT", - /* 90 */ "BOOL", - /* 91 */ "TINYINT", - /* 92 */ "SMALLINT", - /* 93 */ "INT", - /* 94 */ "INTEGER", - /* 95 */ "BIGINT", - /* 96 */ "FLOAT", - /* 97 */ "DOUBLE", - /* 98 */ "BINARY", - /* 99 */ "TIMESTAMP", - /* 100 */ "NCHAR", - /* 101 */ "UNSIGNED", - /* 102 */ "JSON", - /* 103 */ "VARCHAR", - /* 104 */ "MEDIUMBLOB", - /* 105 */ "BLOB", - /* 106 */ "VARBINARY", - /* 107 */ "DECIMAL", - /* 108 */ "SMA", - /* 109 */ "ROLLUP", - /* 110 */ "INDEX", - /* 111 */ "FULLTEXT", - /* 112 */ "FUNCTION", - /* 113 */ "INTERVAL", - /* 114 */ "TOPIC", - /* 115 */ "AS", - /* 116 */ "MNODES", - /* 117 */ "NK_BOOL", - /* 118 */ "NK_VARIABLE", - /* 119 */ "BETWEEN", - /* 120 */ "IS", - /* 121 */ "NULL", - /* 122 */ "NK_LT", - /* 123 */ "NK_GT", - /* 124 */ "NK_LE", - /* 125 */ "NK_GE", - /* 126 */ "NK_NE", - /* 127 */ "LIKE", - /* 128 */ "MATCH", - /* 129 */ "NMATCH", - /* 130 */ "IN", - /* 131 */ "FROM", - /* 132 */ "JOIN", - /* 133 */ "INNER", - /* 134 */ "SELECT", - /* 135 */ "DISTINCT", - /* 136 */ "WHERE", - /* 137 */ "PARTITION", - /* 138 */ "BY", - /* 139 */ "SESSION", - /* 140 */ "STATE_WINDOW", - /* 141 */ "SLIDING", - /* 142 */ "FILL", - /* 143 */ "VALUE", - /* 144 */ "NONE", - /* 145 */ "PREV", - /* 146 */ "LINEAR", - /* 147 */ "NEXT", - /* 148 */ "GROUP", - /* 149 */ "HAVING", - /* 150 */ "ORDER", - /* 151 */ "SLIMIT", - /* 152 */ "SOFFSET", - /* 153 */ "LIMIT", - /* 154 */ "OFFSET", - /* 155 */ "ASC", - /* 156 */ "DESC", - /* 157 */ "NULLS", - /* 158 */ "FIRST", - /* 159 */ "LAST", - /* 160 */ "cmd", - /* 161 */ "account_options", - /* 162 */ "alter_account_options", - /* 163 */ "literal", - /* 164 */ "alter_account_option", - /* 165 */ "user_name", - /* 166 */ "dnode_endpoint", - /* 167 */ "dnode_host_name", - /* 168 */ "not_exists_opt", - /* 169 */ "db_name", - /* 170 */ "db_options", - /* 171 */ "exists_opt", - /* 172 */ "alter_db_options", - /* 173 */ "alter_db_option", - /* 174 */ "full_table_name", - /* 175 */ "column_def_list", - /* 176 */ "tags_def_opt", - /* 177 */ "table_options", - /* 178 */ "multi_create_clause", - /* 179 */ "tags_def", - /* 180 */ "multi_drop_clause", - /* 181 */ "alter_table_clause", - /* 182 */ "alter_table_options", - /* 183 */ "column_name", - /* 184 */ "type_name", - /* 185 */ "create_subtable_clause", - /* 186 */ "specific_tags_opt", - /* 187 */ "literal_list", - /* 188 */ "drop_table_clause", - /* 189 */ "col_name_list", - /* 190 */ "table_name", - /* 191 */ "column_def", - /* 192 */ "func_name_list", - /* 193 */ "alter_table_option", - /* 194 */ "col_name", - /* 195 */ "func_name", - /* 196 */ "function_name", - /* 197 */ "index_name", - /* 198 */ "index_options", - /* 199 */ "func_list", - /* 200 */ "duration_literal", - /* 201 */ "sliding_opt", - /* 202 */ "func", - /* 203 */ "expression_list", - /* 204 */ "topic_name", - /* 205 */ "query_expression", - /* 206 */ "table_alias", - /* 207 */ "column_alias", - /* 208 */ "expression", - /* 209 */ "column_reference", - /* 210 */ "subquery", - /* 211 */ "predicate", - /* 212 */ "compare_op", - /* 213 */ "in_op", - /* 214 */ "in_predicate_value", - /* 215 */ "boolean_value_expression", - /* 216 */ "boolean_primary", - /* 217 */ "common_expression", - /* 218 */ "from_clause", - /* 219 */ "table_reference_list", - /* 220 */ "table_reference", - /* 221 */ "table_primary", - /* 222 */ "joined_table", - /* 223 */ "alias_opt", - /* 224 */ "parenthesized_joined_table", - /* 225 */ "join_type", - /* 226 */ "search_condition", - /* 227 */ "query_specification", - /* 228 */ "set_quantifier_opt", - /* 229 */ "select_list", - /* 230 */ "where_clause_opt", - /* 231 */ "partition_by_clause_opt", - /* 232 */ "twindow_clause_opt", - /* 233 */ "group_by_clause_opt", - /* 234 */ "having_clause_opt", - /* 235 */ "select_sublist", - /* 236 */ "select_item", - /* 237 */ "fill_opt", - /* 238 */ "fill_mode", - /* 239 */ "group_by_list", - /* 240 */ "query_expression_body", - /* 241 */ "order_by_clause_opt", - /* 242 */ "slimit_clause_opt", - /* 243 */ "limit_clause_opt", - /* 244 */ "query_primary", - /* 245 */ "sort_specification_list", - /* 246 */ "sort_specification", - /* 247 */ "ordering_specification_opt", - /* 248 */ "null_ordering_opt", + /* 36 */ "DNODE", + /* 37 */ "PORT", + /* 38 */ "NK_INTEGER", + /* 39 */ "DNODES", + /* 40 */ "NK_IPTOKEN", + /* 41 */ "LOCAL", + /* 42 */ "QNODE", + /* 43 */ "ON", + /* 44 */ "DATABASE", + /* 45 */ "USE", + /* 46 */ "IF", + /* 47 */ "NOT", + /* 48 */ "EXISTS", + /* 49 */ "BLOCKS", + /* 50 */ "CACHE", + /* 51 */ "CACHELAST", + /* 52 */ "COMP", + /* 53 */ "DAYS", + /* 54 */ "FSYNC", + /* 55 */ "MAXROWS", + /* 56 */ "MINROWS", + /* 57 */ "KEEP", + /* 58 */ "PRECISION", + /* 59 */ "QUORUM", + /* 60 */ "REPLICA", + /* 61 */ "TTL", + /* 62 */ "WAL", + /* 63 */ "VGROUPS", + /* 64 */ "SINGLE_STABLE", + /* 65 */ "STREAM_MODE", + /* 66 */ "RETENTIONS", + /* 67 */ "FILE_FACTOR", + /* 68 */ "NK_FLOAT", + /* 69 */ "TABLE", + /* 70 */ "NK_LP", + /* 71 */ "NK_RP", + /* 72 */ "STABLE", + /* 73 */ "ADD", + /* 74 */ "COLUMN", + /* 75 */ "MODIFY", + /* 76 */ "RENAME", + /* 77 */ "TAG", + /* 78 */ "SET", + /* 79 */ "NK_EQ", + /* 80 */ "USING", + /* 81 */ "TAGS", + /* 82 */ "NK_DOT", + /* 83 */ "NK_COMMA", + /* 84 */ "COMMENT", + /* 85 */ "BOOL", + /* 86 */ "TINYINT", + /* 87 */ "SMALLINT", + /* 88 */ "INT", + /* 89 */ "INTEGER", + /* 90 */ "BIGINT", + /* 91 */ "FLOAT", + /* 92 */ "DOUBLE", + /* 93 */ "BINARY", + /* 94 */ "TIMESTAMP", + /* 95 */ "NCHAR", + /* 96 */ "UNSIGNED", + /* 97 */ "JSON", + /* 98 */ "VARCHAR", + /* 99 */ "MEDIUMBLOB", + /* 100 */ "BLOB", + /* 101 */ "VARBINARY", + /* 102 */ "DECIMAL", + /* 103 */ "SMA", + /* 104 */ "ROLLUP", + /* 105 */ "SHOW", + /* 106 */ "DATABASES", + /* 107 */ "TABLES", + /* 108 */ "STABLES", + /* 109 */ "MNODES", + /* 110 */ "MODULES", + /* 111 */ "QNODES", + /* 112 */ "FUNCTIONS", + /* 113 */ "INDEXES", + /* 114 */ "FROM", + /* 115 */ "LIKE", + /* 116 */ "INDEX", + /* 117 */ "FULLTEXT", + /* 118 */ "FUNCTION", + /* 119 */ "INTERVAL", + /* 120 */ "TOPIC", + /* 121 */ "AS", + /* 122 */ "NK_BOOL", + /* 123 */ "NK_VARIABLE", + /* 124 */ "BETWEEN", + /* 125 */ "IS", + /* 126 */ "NULL", + /* 127 */ "NK_LT", + /* 128 */ "NK_GT", + /* 129 */ "NK_LE", + /* 130 */ "NK_GE", + /* 131 */ "NK_NE", + /* 132 */ "MATCH", + /* 133 */ "NMATCH", + /* 134 */ "IN", + /* 135 */ "JOIN", + /* 136 */ "INNER", + /* 137 */ "SELECT", + /* 138 */ "DISTINCT", + /* 139 */ "WHERE", + /* 140 */ "PARTITION", + /* 141 */ "BY", + /* 142 */ "SESSION", + /* 143 */ "STATE_WINDOW", + /* 144 */ "SLIDING", + /* 145 */ "FILL", + /* 146 */ "VALUE", + /* 147 */ "NONE", + /* 148 */ "PREV", + /* 149 */ "LINEAR", + /* 150 */ "NEXT", + /* 151 */ "GROUP", + /* 152 */ "HAVING", + /* 153 */ "ORDER", + /* 154 */ "SLIMIT", + /* 155 */ "SOFFSET", + /* 156 */ "LIMIT", + /* 157 */ "OFFSET", + /* 158 */ "ASC", + /* 159 */ "DESC", + /* 160 */ "NULLS", + /* 161 */ "FIRST", + /* 162 */ "LAST", + /* 163 */ "cmd", + /* 164 */ "account_options", + /* 165 */ "alter_account_options", + /* 166 */ "literal", + /* 167 */ "alter_account_option", + /* 168 */ "user_name", + /* 169 */ "dnode_endpoint", + /* 170 */ "dnode_host_name", + /* 171 */ "not_exists_opt", + /* 172 */ "db_name", + /* 173 */ "db_options", + /* 174 */ "exists_opt", + /* 175 */ "alter_db_options", + /* 176 */ "alter_db_option", + /* 177 */ "full_table_name", + /* 178 */ "column_def_list", + /* 179 */ "tags_def_opt", + /* 180 */ "table_options", + /* 181 */ "multi_create_clause", + /* 182 */ "tags_def", + /* 183 */ "multi_drop_clause", + /* 184 */ "alter_table_clause", + /* 185 */ "alter_table_options", + /* 186 */ "column_name", + /* 187 */ "type_name", + /* 188 */ "create_subtable_clause", + /* 189 */ "specific_tags_opt", + /* 190 */ "literal_list", + /* 191 */ "drop_table_clause", + /* 192 */ "col_name_list", + /* 193 */ "table_name", + /* 194 */ "column_def", + /* 195 */ "func_name_list", + /* 196 */ "alter_table_option", + /* 197 */ "col_name", + /* 198 */ "db_name_cond_opt", + /* 199 */ "like_pattern_opt", + /* 200 */ "table_name_cond", + /* 201 */ "from_db_opt", + /* 202 */ "func_name", + /* 203 */ "function_name", + /* 204 */ "index_name", + /* 205 */ "index_options", + /* 206 */ "func_list", + /* 207 */ "duration_literal", + /* 208 */ "sliding_opt", + /* 209 */ "func", + /* 210 */ "expression_list", + /* 211 */ "topic_name", + /* 212 */ "query_expression", + /* 213 */ "signed", + /* 214 */ "signed_literal", + /* 215 */ "table_alias", + /* 216 */ "column_alias", + /* 217 */ "expression", + /* 218 */ "column_reference", + /* 219 */ "subquery", + /* 220 */ "predicate", + /* 221 */ "compare_op", + /* 222 */ "in_op", + /* 223 */ "in_predicate_value", + /* 224 */ "boolean_value_expression", + /* 225 */ "boolean_primary", + /* 226 */ "common_expression", + /* 227 */ "from_clause", + /* 228 */ "table_reference_list", + /* 229 */ "table_reference", + /* 230 */ "table_primary", + /* 231 */ "joined_table", + /* 232 */ "alias_opt", + /* 233 */ "parenthesized_joined_table", + /* 234 */ "join_type", + /* 235 */ "search_condition", + /* 236 */ "query_specification", + /* 237 */ "set_quantifier_opt", + /* 238 */ "select_list", + /* 239 */ "where_clause_opt", + /* 240 */ "partition_by_clause_opt", + /* 241 */ "twindow_clause_opt", + /* 242 */ "group_by_clause_opt", + /* 243 */ "having_clause_opt", + /* 244 */ "select_sublist", + /* 245 */ "select_item", + /* 246 */ "fill_opt", + /* 247 */ "fill_mode", + /* 248 */ "group_by_list", + /* 249 */ "query_expression_body", + /* 250 */ "order_by_clause_opt", + /* 251 */ "slimit_clause_opt", + /* 252 */ "limit_clause_opt", + /* 253 */ "query_primary", + /* 254 */ "sort_specification_list", + /* 255 */ "sort_specification", + /* 256 */ "ordering_specification_opt", + /* 257 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -971,294 +999,315 @@ static const char *const yyRuleName[] = { /* 25 */ "cmd ::= ALTER USER user_name PASS NK_STRING", /* 26 */ "cmd ::= ALTER USER user_name PRIVILEGE NK_STRING", /* 27 */ "cmd ::= DROP USER user_name", - /* 28 */ "cmd ::= SHOW USERS", - /* 29 */ "cmd ::= CREATE DNODE dnode_endpoint", - /* 30 */ "cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER", - /* 31 */ "cmd ::= DROP DNODE NK_INTEGER", - /* 32 */ "cmd ::= DROP DNODE dnode_endpoint", - /* 33 */ "cmd ::= SHOW DNODES", - /* 34 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", - /* 35 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", - /* 36 */ "cmd ::= ALTER ALL DNODES NK_STRING", - /* 37 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", - /* 38 */ "dnode_endpoint ::= NK_STRING", - /* 39 */ "dnode_host_name ::= NK_ID", - /* 40 */ "dnode_host_name ::= NK_IPTOKEN", - /* 41 */ "cmd ::= ALTER LOCAL NK_STRING", - /* 42 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", - /* 43 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", - /* 44 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", - /* 45 */ "cmd ::= SHOW QNODES", - /* 46 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", - /* 47 */ "cmd ::= DROP DATABASE exists_opt db_name", - /* 48 */ "cmd ::= SHOW DATABASES", - /* 49 */ "cmd ::= USE db_name", - /* 50 */ "cmd ::= ALTER DATABASE db_name alter_db_options", - /* 51 */ "not_exists_opt ::= IF NOT EXISTS", - /* 52 */ "not_exists_opt ::=", - /* 53 */ "exists_opt ::= IF EXISTS", - /* 54 */ "exists_opt ::=", - /* 55 */ "db_options ::=", - /* 56 */ "db_options ::= db_options BLOCKS NK_INTEGER", - /* 57 */ "db_options ::= db_options CACHE NK_INTEGER", - /* 58 */ "db_options ::= db_options CACHELAST NK_INTEGER", - /* 59 */ "db_options ::= db_options COMP NK_INTEGER", - /* 60 */ "db_options ::= db_options DAYS NK_INTEGER", - /* 61 */ "db_options ::= db_options FSYNC NK_INTEGER", - /* 62 */ "db_options ::= db_options MAXROWS NK_INTEGER", - /* 63 */ "db_options ::= db_options MINROWS NK_INTEGER", - /* 64 */ "db_options ::= db_options KEEP NK_INTEGER", - /* 65 */ "db_options ::= db_options PRECISION NK_STRING", - /* 66 */ "db_options ::= db_options QUORUM NK_INTEGER", - /* 67 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 68 */ "db_options ::= db_options TTL NK_INTEGER", - /* 69 */ "db_options ::= db_options WAL NK_INTEGER", - /* 70 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 71 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 72 */ "db_options ::= db_options STREAM_MODE NK_INTEGER", - /* 73 */ "db_options ::= db_options RETENTIONS NK_STRING", - /* 74 */ "db_options ::= db_options FILE_FACTOR NK_FLOAT", - /* 75 */ "alter_db_options ::= alter_db_option", - /* 76 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 77 */ "alter_db_option ::= BLOCKS NK_INTEGER", - /* 78 */ "alter_db_option ::= FSYNC NK_INTEGER", - /* 79 */ "alter_db_option ::= KEEP NK_INTEGER", - /* 80 */ "alter_db_option ::= WAL NK_INTEGER", - /* 81 */ "alter_db_option ::= QUORUM NK_INTEGER", - /* 82 */ "alter_db_option ::= CACHELAST NK_INTEGER", - /* 83 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 84 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 85 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 86 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 87 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 88 */ "cmd ::= SHOW TABLES", - /* 89 */ "cmd ::= SHOW STABLES", - /* 90 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 91 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 92 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 93 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 94 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 95 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 96 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 97 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 98 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 99 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 100 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 101 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal", - /* 102 */ "multi_create_clause ::= create_subtable_clause", - /* 103 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 104 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP", - /* 105 */ "multi_drop_clause ::= drop_table_clause", - /* 106 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", - /* 107 */ "drop_table_clause ::= exists_opt full_table_name", - /* 108 */ "specific_tags_opt ::=", - /* 109 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", - /* 110 */ "full_table_name ::= table_name", - /* 111 */ "full_table_name ::= db_name NK_DOT table_name", - /* 112 */ "column_def_list ::= column_def", - /* 113 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 114 */ "column_def ::= column_name type_name", - /* 115 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 116 */ "type_name ::= BOOL", - /* 117 */ "type_name ::= TINYINT", - /* 118 */ "type_name ::= SMALLINT", - /* 119 */ "type_name ::= INT", - /* 120 */ "type_name ::= INTEGER", - /* 121 */ "type_name ::= BIGINT", - /* 122 */ "type_name ::= FLOAT", - /* 123 */ "type_name ::= DOUBLE", - /* 124 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 125 */ "type_name ::= TIMESTAMP", - /* 126 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 127 */ "type_name ::= TINYINT UNSIGNED", - /* 128 */ "type_name ::= SMALLINT UNSIGNED", - /* 129 */ "type_name ::= INT UNSIGNED", - /* 130 */ "type_name ::= BIGINT UNSIGNED", - /* 131 */ "type_name ::= JSON", - /* 132 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 133 */ "type_name ::= MEDIUMBLOB", - /* 134 */ "type_name ::= BLOB", - /* 135 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 136 */ "type_name ::= DECIMAL", - /* 137 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 138 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 139 */ "tags_def_opt ::=", - /* 140 */ "tags_def_opt ::= tags_def", - /* 141 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 142 */ "table_options ::=", - /* 143 */ "table_options ::= table_options COMMENT NK_STRING", - /* 144 */ "table_options ::= table_options KEEP NK_INTEGER", - /* 145 */ "table_options ::= table_options TTL NK_INTEGER", - /* 146 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 147 */ "table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP", - /* 148 */ "alter_table_options ::= alter_table_option", - /* 149 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 150 */ "alter_table_option ::= COMMENT NK_STRING", - /* 151 */ "alter_table_option ::= KEEP NK_INTEGER", - /* 152 */ "alter_table_option ::= TTL NK_INTEGER", - /* 153 */ "col_name_list ::= col_name", - /* 154 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 155 */ "col_name ::= column_name", - /* 156 */ "func_name_list ::= func_name", - /* 157 */ "func_name_list ::= func_name_list NK_COMMA col_name", - /* 158 */ "func_name ::= function_name", - /* 159 */ "cmd ::= CREATE SMA INDEX index_name ON table_name index_options", - /* 160 */ "cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP", - /* 161 */ "cmd ::= DROP INDEX index_name ON table_name", - /* 162 */ "index_options ::=", - /* 163 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", - /* 164 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", - /* 165 */ "func_list ::= func", - /* 166 */ "func_list ::= func_list NK_COMMA func", - /* 167 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 168 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", - /* 169 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name", - /* 170 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 171 */ "cmd ::= SHOW VGROUPS", - /* 172 */ "cmd ::= SHOW db_name NK_DOT VGROUPS", - /* 173 */ "cmd ::= SHOW MNODES", - /* 174 */ "cmd ::= query_expression", - /* 175 */ "literal ::= NK_INTEGER", - /* 176 */ "literal ::= NK_FLOAT", - /* 177 */ "literal ::= NK_STRING", - /* 178 */ "literal ::= NK_BOOL", - /* 179 */ "literal ::= TIMESTAMP NK_STRING", - /* 180 */ "literal ::= duration_literal", - /* 181 */ "duration_literal ::= NK_VARIABLE", - /* 182 */ "literal_list ::= literal", - /* 183 */ "literal_list ::= literal_list NK_COMMA literal", - /* 184 */ "db_name ::= NK_ID", - /* 185 */ "table_name ::= NK_ID", - /* 186 */ "column_name ::= NK_ID", - /* 187 */ "function_name ::= NK_ID", - /* 188 */ "table_alias ::= NK_ID", - /* 189 */ "column_alias ::= NK_ID", - /* 190 */ "user_name ::= NK_ID", - /* 191 */ "index_name ::= NK_ID", - /* 192 */ "topic_name ::= NK_ID", - /* 193 */ "expression ::= literal", - /* 194 */ "expression ::= column_reference", - /* 195 */ "expression ::= function_name NK_LP expression_list NK_RP", - /* 196 */ "expression ::= function_name NK_LP NK_STAR NK_RP", - /* 197 */ "expression ::= subquery", - /* 198 */ "expression ::= NK_LP expression NK_RP", - /* 199 */ "expression ::= NK_PLUS expression", - /* 200 */ "expression ::= NK_MINUS expression", - /* 201 */ "expression ::= expression NK_PLUS expression", - /* 202 */ "expression ::= expression NK_MINUS expression", - /* 203 */ "expression ::= expression NK_STAR expression", - /* 204 */ "expression ::= expression NK_SLASH expression", - /* 205 */ "expression ::= expression NK_REM expression", - /* 206 */ "expression_list ::= expression", - /* 207 */ "expression_list ::= expression_list NK_COMMA expression", - /* 208 */ "column_reference ::= column_name", - /* 209 */ "column_reference ::= table_name NK_DOT column_name", - /* 210 */ "predicate ::= expression compare_op expression", - /* 211 */ "predicate ::= expression BETWEEN expression AND expression", - /* 212 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 213 */ "predicate ::= expression IS NULL", - /* 214 */ "predicate ::= expression IS NOT NULL", - /* 215 */ "predicate ::= expression in_op in_predicate_value", - /* 216 */ "compare_op ::= NK_LT", - /* 217 */ "compare_op ::= NK_GT", - /* 218 */ "compare_op ::= NK_LE", - /* 219 */ "compare_op ::= NK_GE", - /* 220 */ "compare_op ::= NK_NE", - /* 221 */ "compare_op ::= NK_EQ", - /* 222 */ "compare_op ::= LIKE", - /* 223 */ "compare_op ::= NOT LIKE", - /* 224 */ "compare_op ::= MATCH", - /* 225 */ "compare_op ::= NMATCH", - /* 226 */ "in_op ::= IN", - /* 227 */ "in_op ::= NOT IN", - /* 228 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 229 */ "boolean_value_expression ::= boolean_primary", - /* 230 */ "boolean_value_expression ::= NOT boolean_primary", - /* 231 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 232 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 233 */ "boolean_primary ::= predicate", - /* 234 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 235 */ "common_expression ::= expression", - /* 236 */ "common_expression ::= boolean_value_expression", - /* 237 */ "from_clause ::= FROM table_reference_list", - /* 238 */ "table_reference_list ::= table_reference", - /* 239 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 240 */ "table_reference ::= table_primary", - /* 241 */ "table_reference ::= joined_table", - /* 242 */ "table_primary ::= table_name alias_opt", - /* 243 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 244 */ "table_primary ::= subquery alias_opt", - /* 245 */ "table_primary ::= parenthesized_joined_table", - /* 246 */ "alias_opt ::=", - /* 247 */ "alias_opt ::= table_alias", - /* 248 */ "alias_opt ::= AS table_alias", - /* 249 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 250 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 251 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 252 */ "join_type ::=", - /* 253 */ "join_type ::= INNER", - /* 254 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 255 */ "set_quantifier_opt ::=", - /* 256 */ "set_quantifier_opt ::= DISTINCT", - /* 257 */ "set_quantifier_opt ::= ALL", - /* 258 */ "select_list ::= NK_STAR", - /* 259 */ "select_list ::= select_sublist", - /* 260 */ "select_sublist ::= select_item", - /* 261 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 262 */ "select_item ::= common_expression", - /* 263 */ "select_item ::= common_expression column_alias", - /* 264 */ "select_item ::= common_expression AS column_alias", - /* 265 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 266 */ "where_clause_opt ::=", - /* 267 */ "where_clause_opt ::= WHERE search_condition", - /* 268 */ "partition_by_clause_opt ::=", - /* 269 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 270 */ "twindow_clause_opt ::=", - /* 271 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", - /* 272 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", - /* 273 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 274 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 275 */ "sliding_opt ::=", - /* 276 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 277 */ "fill_opt ::=", - /* 278 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 279 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 280 */ "fill_mode ::= NONE", - /* 281 */ "fill_mode ::= PREV", - /* 282 */ "fill_mode ::= NULL", - /* 283 */ "fill_mode ::= LINEAR", - /* 284 */ "fill_mode ::= NEXT", - /* 285 */ "group_by_clause_opt ::=", - /* 286 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 287 */ "group_by_list ::= expression", - /* 288 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 289 */ "having_clause_opt ::=", - /* 290 */ "having_clause_opt ::= HAVING search_condition", - /* 291 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 292 */ "query_expression_body ::= query_primary", - /* 293 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 294 */ "query_primary ::= query_specification", - /* 295 */ "order_by_clause_opt ::=", - /* 296 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 297 */ "slimit_clause_opt ::=", - /* 298 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 299 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 300 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 301 */ "limit_clause_opt ::=", - /* 302 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 303 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 304 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 305 */ "subquery ::= NK_LP query_expression NK_RP", - /* 306 */ "search_condition ::= common_expression", - /* 307 */ "sort_specification_list ::= sort_specification", - /* 308 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 309 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 310 */ "ordering_specification_opt ::=", - /* 311 */ "ordering_specification_opt ::= ASC", - /* 312 */ "ordering_specification_opt ::= DESC", - /* 313 */ "null_ordering_opt ::=", - /* 314 */ "null_ordering_opt ::= NULLS FIRST", - /* 315 */ "null_ordering_opt ::= NULLS LAST", + /* 28 */ "cmd ::= CREATE DNODE dnode_endpoint", + /* 29 */ "cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER", + /* 30 */ "cmd ::= DROP DNODE NK_INTEGER", + /* 31 */ "cmd ::= DROP DNODE dnode_endpoint", + /* 32 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", + /* 33 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", + /* 34 */ "cmd ::= ALTER ALL DNODES NK_STRING", + /* 35 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", + /* 36 */ "dnode_endpoint ::= NK_STRING", + /* 37 */ "dnode_host_name ::= NK_ID", + /* 38 */ "dnode_host_name ::= NK_IPTOKEN", + /* 39 */ "cmd ::= ALTER LOCAL NK_STRING", + /* 40 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", + /* 41 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", + /* 42 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", + /* 43 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", + /* 44 */ "cmd ::= DROP DATABASE exists_opt db_name", + /* 45 */ "cmd ::= USE db_name", + /* 46 */ "cmd ::= ALTER DATABASE db_name alter_db_options", + /* 47 */ "not_exists_opt ::= IF NOT EXISTS", + /* 48 */ "not_exists_opt ::=", + /* 49 */ "exists_opt ::= IF EXISTS", + /* 50 */ "exists_opt ::=", + /* 51 */ "db_options ::=", + /* 52 */ "db_options ::= db_options BLOCKS NK_INTEGER", + /* 53 */ "db_options ::= db_options CACHE NK_INTEGER", + /* 54 */ "db_options ::= db_options CACHELAST NK_INTEGER", + /* 55 */ "db_options ::= db_options COMP NK_INTEGER", + /* 56 */ "db_options ::= db_options DAYS NK_INTEGER", + /* 57 */ "db_options ::= db_options FSYNC NK_INTEGER", + /* 58 */ "db_options ::= db_options MAXROWS NK_INTEGER", + /* 59 */ "db_options ::= db_options MINROWS NK_INTEGER", + /* 60 */ "db_options ::= db_options KEEP NK_INTEGER", + /* 61 */ "db_options ::= db_options PRECISION NK_STRING", + /* 62 */ "db_options ::= db_options QUORUM NK_INTEGER", + /* 63 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 64 */ "db_options ::= db_options TTL NK_INTEGER", + /* 65 */ "db_options ::= db_options WAL NK_INTEGER", + /* 66 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 67 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 68 */ "db_options ::= db_options STREAM_MODE NK_INTEGER", + /* 69 */ "db_options ::= db_options RETENTIONS NK_STRING", + /* 70 */ "db_options ::= db_options FILE_FACTOR NK_FLOAT", + /* 71 */ "alter_db_options ::= alter_db_option", + /* 72 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 73 */ "alter_db_option ::= BLOCKS NK_INTEGER", + /* 74 */ "alter_db_option ::= FSYNC NK_INTEGER", + /* 75 */ "alter_db_option ::= KEEP NK_INTEGER", + /* 76 */ "alter_db_option ::= WAL NK_INTEGER", + /* 77 */ "alter_db_option ::= QUORUM NK_INTEGER", + /* 78 */ "alter_db_option ::= CACHELAST NK_INTEGER", + /* 79 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 80 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 81 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 82 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 83 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 84 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 85 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 86 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 87 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 88 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 89 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 90 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 91 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 92 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 93 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 94 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 95 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal", + /* 96 */ "multi_create_clause ::= create_subtable_clause", + /* 97 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 98 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP", + /* 99 */ "multi_drop_clause ::= drop_table_clause", + /* 100 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", + /* 101 */ "drop_table_clause ::= exists_opt full_table_name", + /* 102 */ "specific_tags_opt ::=", + /* 103 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", + /* 104 */ "full_table_name ::= table_name", + /* 105 */ "full_table_name ::= db_name NK_DOT table_name", + /* 106 */ "column_def_list ::= column_def", + /* 107 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 108 */ "column_def ::= column_name type_name", + /* 109 */ "column_def ::= column_name type_name COMMENT NK_STRING", + /* 110 */ "type_name ::= BOOL", + /* 111 */ "type_name ::= TINYINT", + /* 112 */ "type_name ::= SMALLINT", + /* 113 */ "type_name ::= INT", + /* 114 */ "type_name ::= INTEGER", + /* 115 */ "type_name ::= BIGINT", + /* 116 */ "type_name ::= FLOAT", + /* 117 */ "type_name ::= DOUBLE", + /* 118 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 119 */ "type_name ::= TIMESTAMP", + /* 120 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 121 */ "type_name ::= TINYINT UNSIGNED", + /* 122 */ "type_name ::= SMALLINT UNSIGNED", + /* 123 */ "type_name ::= INT UNSIGNED", + /* 124 */ "type_name ::= BIGINT UNSIGNED", + /* 125 */ "type_name ::= JSON", + /* 126 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 127 */ "type_name ::= MEDIUMBLOB", + /* 128 */ "type_name ::= BLOB", + /* 129 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 130 */ "type_name ::= DECIMAL", + /* 131 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 132 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 133 */ "tags_def_opt ::=", + /* 134 */ "tags_def_opt ::= tags_def", + /* 135 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 136 */ "table_options ::=", + /* 137 */ "table_options ::= table_options COMMENT NK_STRING", + /* 138 */ "table_options ::= table_options KEEP NK_INTEGER", + /* 139 */ "table_options ::= table_options TTL NK_INTEGER", + /* 140 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 141 */ "table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP", + /* 142 */ "alter_table_options ::= alter_table_option", + /* 143 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 144 */ "alter_table_option ::= COMMENT NK_STRING", + /* 145 */ "alter_table_option ::= KEEP NK_INTEGER", + /* 146 */ "alter_table_option ::= TTL NK_INTEGER", + /* 147 */ "col_name_list ::= col_name", + /* 148 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 149 */ "col_name ::= column_name", + /* 150 */ "cmd ::= SHOW DNODES", + /* 151 */ "cmd ::= SHOW USERS", + /* 152 */ "cmd ::= SHOW DATABASES", + /* 153 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 154 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 155 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 156 */ "cmd ::= SHOW MNODES", + /* 157 */ "cmd ::= SHOW MODULES", + /* 158 */ "cmd ::= SHOW QNODES", + /* 159 */ "cmd ::= SHOW FUNCTIONS", + /* 160 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 161 */ "cmd ::= SHOW STREAMS", + /* 162 */ "db_name_cond_opt ::=", + /* 163 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 164 */ "like_pattern_opt ::=", + /* 165 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 166 */ "table_name_cond ::= table_name", + /* 167 */ "from_db_opt ::=", + /* 168 */ "from_db_opt ::= FROM db_name", + /* 169 */ "func_name_list ::= func_name", + /* 170 */ "func_name_list ::= func_name_list NK_COMMA col_name", + /* 171 */ "func_name ::= function_name", + /* 172 */ "cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options", + /* 173 */ "cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP", + /* 174 */ "cmd ::= DROP INDEX exists_opt index_name ON table_name", + /* 175 */ "index_options ::=", + /* 176 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", + /* 177 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", + /* 178 */ "func_list ::= func", + /* 179 */ "func_list ::= func_list NK_COMMA func", + /* 180 */ "func ::= function_name NK_LP expression_list NK_RP", + /* 181 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", + /* 182 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name", + /* 183 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 184 */ "cmd ::= query_expression", + /* 185 */ "literal ::= NK_INTEGER", + /* 186 */ "literal ::= NK_FLOAT", + /* 187 */ "literal ::= NK_STRING", + /* 188 */ "literal ::= NK_BOOL", + /* 189 */ "literal ::= TIMESTAMP NK_STRING", + /* 190 */ "literal ::= duration_literal", + /* 191 */ "duration_literal ::= NK_VARIABLE", + /* 192 */ "signed ::= NK_INTEGER", + /* 193 */ "signed ::= NK_PLUS NK_INTEGER", + /* 194 */ "signed ::= NK_MINUS NK_INTEGER", + /* 195 */ "signed ::= NK_FLOAT", + /* 196 */ "signed ::= NK_PLUS NK_FLOAT", + /* 197 */ "signed ::= NK_MINUS NK_FLOAT", + /* 198 */ "signed_literal ::= signed", + /* 199 */ "signed_literal ::= NK_STRING", + /* 200 */ "signed_literal ::= NK_BOOL", + /* 201 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 202 */ "signed_literal ::= duration_literal", + /* 203 */ "literal_list ::= signed_literal", + /* 204 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 205 */ "db_name ::= NK_ID", + /* 206 */ "table_name ::= NK_ID", + /* 207 */ "column_name ::= NK_ID", + /* 208 */ "function_name ::= NK_ID", + /* 209 */ "table_alias ::= NK_ID", + /* 210 */ "column_alias ::= NK_ID", + /* 211 */ "user_name ::= NK_ID", + /* 212 */ "index_name ::= NK_ID", + /* 213 */ "topic_name ::= NK_ID", + /* 214 */ "expression ::= literal", + /* 215 */ "expression ::= column_reference", + /* 216 */ "expression ::= function_name NK_LP expression_list NK_RP", + /* 217 */ "expression ::= function_name NK_LP NK_STAR NK_RP", + /* 218 */ "expression ::= subquery", + /* 219 */ "expression ::= NK_LP expression NK_RP", + /* 220 */ "expression ::= NK_PLUS expression", + /* 221 */ "expression ::= NK_MINUS expression", + /* 222 */ "expression ::= expression NK_PLUS expression", + /* 223 */ "expression ::= expression NK_MINUS expression", + /* 224 */ "expression ::= expression NK_STAR expression", + /* 225 */ "expression ::= expression NK_SLASH expression", + /* 226 */ "expression ::= expression NK_REM expression", + /* 227 */ "expression_list ::= expression", + /* 228 */ "expression_list ::= expression_list NK_COMMA expression", + /* 229 */ "column_reference ::= column_name", + /* 230 */ "column_reference ::= table_name NK_DOT column_name", + /* 231 */ "predicate ::= expression compare_op expression", + /* 232 */ "predicate ::= expression BETWEEN expression AND expression", + /* 233 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 234 */ "predicate ::= expression IS NULL", + /* 235 */ "predicate ::= expression IS NOT NULL", + /* 236 */ "predicate ::= expression in_op in_predicate_value", + /* 237 */ "compare_op ::= NK_LT", + /* 238 */ "compare_op ::= NK_GT", + /* 239 */ "compare_op ::= NK_LE", + /* 240 */ "compare_op ::= NK_GE", + /* 241 */ "compare_op ::= NK_NE", + /* 242 */ "compare_op ::= NK_EQ", + /* 243 */ "compare_op ::= LIKE", + /* 244 */ "compare_op ::= NOT LIKE", + /* 245 */ "compare_op ::= MATCH", + /* 246 */ "compare_op ::= NMATCH", + /* 247 */ "in_op ::= IN", + /* 248 */ "in_op ::= NOT IN", + /* 249 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 250 */ "boolean_value_expression ::= boolean_primary", + /* 251 */ "boolean_value_expression ::= NOT boolean_primary", + /* 252 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 253 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 254 */ "boolean_primary ::= predicate", + /* 255 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 256 */ "common_expression ::= expression", + /* 257 */ "common_expression ::= boolean_value_expression", + /* 258 */ "from_clause ::= FROM table_reference_list", + /* 259 */ "table_reference_list ::= table_reference", + /* 260 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 261 */ "table_reference ::= table_primary", + /* 262 */ "table_reference ::= joined_table", + /* 263 */ "table_primary ::= table_name alias_opt", + /* 264 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 265 */ "table_primary ::= subquery alias_opt", + /* 266 */ "table_primary ::= parenthesized_joined_table", + /* 267 */ "alias_opt ::=", + /* 268 */ "alias_opt ::= table_alias", + /* 269 */ "alias_opt ::= AS table_alias", + /* 270 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 271 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 272 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 273 */ "join_type ::=", + /* 274 */ "join_type ::= INNER", + /* 275 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 276 */ "set_quantifier_opt ::=", + /* 277 */ "set_quantifier_opt ::= DISTINCT", + /* 278 */ "set_quantifier_opt ::= ALL", + /* 279 */ "select_list ::= NK_STAR", + /* 280 */ "select_list ::= select_sublist", + /* 281 */ "select_sublist ::= select_item", + /* 282 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 283 */ "select_item ::= common_expression", + /* 284 */ "select_item ::= common_expression column_alias", + /* 285 */ "select_item ::= common_expression AS column_alias", + /* 286 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 287 */ "where_clause_opt ::=", + /* 288 */ "where_clause_opt ::= WHERE search_condition", + /* 289 */ "partition_by_clause_opt ::=", + /* 290 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 291 */ "twindow_clause_opt ::=", + /* 292 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 293 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 294 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 295 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 296 */ "sliding_opt ::=", + /* 297 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 298 */ "fill_opt ::=", + /* 299 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 300 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 301 */ "fill_mode ::= NONE", + /* 302 */ "fill_mode ::= PREV", + /* 303 */ "fill_mode ::= NULL", + /* 304 */ "fill_mode ::= LINEAR", + /* 305 */ "fill_mode ::= NEXT", + /* 306 */ "group_by_clause_opt ::=", + /* 307 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 308 */ "group_by_list ::= expression", + /* 309 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 310 */ "having_clause_opt ::=", + /* 311 */ "having_clause_opt ::= HAVING search_condition", + /* 312 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 313 */ "query_expression_body ::= query_primary", + /* 314 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 315 */ "query_primary ::= query_specification", + /* 316 */ "order_by_clause_opt ::=", + /* 317 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 318 */ "slimit_clause_opt ::=", + /* 319 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 320 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 321 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 322 */ "limit_clause_opt ::=", + /* 323 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 324 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 325 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 326 */ "subquery ::= NK_LP query_expression NK_RP", + /* 327 */ "search_condition ::= common_expression", + /* 328 */ "sort_specification_list ::= sort_specification", + /* 329 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 330 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 331 */ "ordering_specification_opt ::=", + /* 332 */ "ordering_specification_opt ::= ASC", + /* 333 */ "ordering_specification_opt ::= DESC", + /* 334 */ "null_ordering_opt ::=", + /* 335 */ "null_ordering_opt ::= NULLS FIRST", + /* 336 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -1276,10 +1325,10 @@ static int yyGrowStack(yyParser *p){ newSize = p->yystksz*2 + 100; idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); + pNew = taosMemoryMalloc(newSize*sizeof(pNew[0])); if( pNew ) pNew[0] = p->yystk0; }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + pNew = taosMemoryRealloc(p->yystack, newSize*sizeof(pNew[0])); } if( pNew ){ p->yystack = pNew; @@ -1385,139 +1434,145 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 160: /* cmd */ - case 163: /* literal */ - case 170: /* db_options */ - case 172: /* alter_db_options */ - case 174: /* full_table_name */ - case 177: /* table_options */ - case 181: /* alter_table_clause */ - case 182: /* alter_table_options */ - case 185: /* create_subtable_clause */ - case 188: /* drop_table_clause */ - case 191: /* column_def */ - case 194: /* col_name */ - case 195: /* func_name */ - case 198: /* index_options */ - case 200: /* duration_literal */ - case 201: /* sliding_opt */ - case 202: /* func */ - case 205: /* query_expression */ - case 208: /* expression */ - case 209: /* column_reference */ - case 210: /* subquery */ - case 211: /* predicate */ - case 214: /* in_predicate_value */ - case 215: /* boolean_value_expression */ - case 216: /* boolean_primary */ - case 217: /* common_expression */ - case 218: /* from_clause */ - case 219: /* table_reference_list */ - case 220: /* table_reference */ - case 221: /* table_primary */ - case 222: /* joined_table */ - case 224: /* parenthesized_joined_table */ - case 226: /* search_condition */ - case 227: /* query_specification */ - case 230: /* where_clause_opt */ - case 232: /* twindow_clause_opt */ - case 234: /* having_clause_opt */ - case 236: /* select_item */ - case 237: /* fill_opt */ - case 240: /* query_expression_body */ - case 242: /* slimit_clause_opt */ - case 243: /* limit_clause_opt */ - case 244: /* query_primary */ - case 246: /* sort_specification */ + case 163: /* cmd */ + case 166: /* literal */ + case 173: /* db_options */ + case 175: /* alter_db_options */ + case 177: /* full_table_name */ + case 180: /* table_options */ + case 184: /* alter_table_clause */ + case 185: /* alter_table_options */ + case 188: /* create_subtable_clause */ + case 191: /* drop_table_clause */ + case 194: /* column_def */ + case 197: /* col_name */ + case 198: /* db_name_cond_opt */ + case 199: /* like_pattern_opt */ + case 200: /* table_name_cond */ + case 201: /* from_db_opt */ + case 202: /* func_name */ + case 205: /* index_options */ + case 207: /* duration_literal */ + case 208: /* sliding_opt */ + case 209: /* func */ + case 212: /* query_expression */ + case 213: /* signed */ + case 214: /* signed_literal */ + case 217: /* expression */ + case 218: /* column_reference */ + case 219: /* subquery */ + case 220: /* predicate */ + case 223: /* in_predicate_value */ + case 224: /* boolean_value_expression */ + case 225: /* boolean_primary */ + case 226: /* common_expression */ + case 227: /* from_clause */ + case 228: /* table_reference_list */ + case 229: /* table_reference */ + case 230: /* table_primary */ + case 231: /* joined_table */ + case 233: /* parenthesized_joined_table */ + case 235: /* search_condition */ + case 236: /* query_specification */ + case 239: /* where_clause_opt */ + case 241: /* twindow_clause_opt */ + case 243: /* having_clause_opt */ + case 245: /* select_item */ + case 246: /* fill_opt */ + case 249: /* query_expression_body */ + case 251: /* slimit_clause_opt */ + case 252: /* limit_clause_opt */ + case 253: /* query_primary */ + case 255: /* sort_specification */ { - nodesDestroyNode((yypminor->yy26)); + nodesDestroyNode((yypminor->yy140)); } break; - case 161: /* account_options */ - case 162: /* alter_account_options */ - case 164: /* alter_account_option */ + case 164: /* account_options */ + case 165: /* alter_account_options */ + case 167: /* alter_account_option */ { } break; - case 165: /* user_name */ - case 166: /* dnode_endpoint */ - case 167: /* dnode_host_name */ - case 169: /* db_name */ - case 183: /* column_name */ - case 190: /* table_name */ - case 196: /* function_name */ - case 197: /* index_name */ - case 204: /* topic_name */ - case 206: /* table_alias */ - case 207: /* column_alias */ - case 223: /* alias_opt */ + case 168: /* user_name */ + case 169: /* dnode_endpoint */ + case 170: /* dnode_host_name */ + case 172: /* db_name */ + case 186: /* column_name */ + case 193: /* table_name */ + case 203: /* function_name */ + case 204: /* index_name */ + case 211: /* topic_name */ + case 215: /* table_alias */ + case 216: /* column_alias */ + case 232: /* alias_opt */ { } break; - case 168: /* not_exists_opt */ - case 171: /* exists_opt */ - case 228: /* set_quantifier_opt */ + case 171: /* not_exists_opt */ + case 174: /* exists_opt */ + case 237: /* set_quantifier_opt */ { } break; - case 173: /* alter_db_option */ - case 193: /* alter_table_option */ + case 176: /* alter_db_option */ + case 196: /* alter_table_option */ { } break; - case 175: /* column_def_list */ - case 176: /* tags_def_opt */ - case 178: /* multi_create_clause */ - case 179: /* tags_def */ - case 180: /* multi_drop_clause */ - case 186: /* specific_tags_opt */ - case 187: /* literal_list */ - case 189: /* col_name_list */ - case 192: /* func_name_list */ - case 199: /* func_list */ - case 203: /* expression_list */ - case 229: /* select_list */ - case 231: /* partition_by_clause_opt */ - case 233: /* group_by_clause_opt */ - case 235: /* select_sublist */ - case 239: /* group_by_list */ - case 241: /* order_by_clause_opt */ - case 245: /* sort_specification_list */ + case 178: /* column_def_list */ + case 179: /* tags_def_opt */ + case 181: /* multi_create_clause */ + case 182: /* tags_def */ + case 183: /* multi_drop_clause */ + case 189: /* specific_tags_opt */ + case 190: /* literal_list */ + case 192: /* col_name_list */ + case 195: /* func_name_list */ + case 206: /* func_list */ + case 210: /* expression_list */ + case 238: /* select_list */ + case 240: /* partition_by_clause_opt */ + case 242: /* group_by_clause_opt */ + case 244: /* select_sublist */ + case 248: /* group_by_list */ + case 250: /* order_by_clause_opt */ + case 254: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy64)); + nodesDestroyList((yypminor->yy136)); } break; - case 184: /* type_name */ + case 187: /* type_name */ { } break; - case 212: /* compare_op */ - case 213: /* in_op */ + case 221: /* compare_op */ + case 222: /* in_op */ { } break; - case 225: /* join_type */ + case 234: /* join_type */ { } break; - case 238: /* fill_mode */ + case 247: /* fill_mode */ { } break; - case 247: /* ordering_specification_opt */ + case 256: /* ordering_specification_opt */ { } break; - case 248: /* null_ordering_opt */ + case 257: /* null_ordering_opt */ { } @@ -1555,7 +1610,7 @@ void ParseFinalize(void *p){ yyParser *pParser = (yyParser*)p; while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); + if( pParser->yystack!=&pParser->yystk0 ) taosMemoryFree(pParser->yystack); #endif } @@ -1816,322 +1871,343 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 160, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 160, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 161, 0 }, /* (2) account_options ::= */ - { 161, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 161, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 161, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 161, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 161, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 161, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 161, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 161, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 161, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 162, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 162, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 164, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 164, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 164, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 164, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 164, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 164, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 164, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 164, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 164, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 164, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 160, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ - { 160, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 160, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ - { 160, -3 }, /* (27) cmd ::= DROP USER user_name */ - { 160, -2 }, /* (28) cmd ::= SHOW USERS */ - { 160, -3 }, /* (29) cmd ::= CREATE DNODE dnode_endpoint */ - { 160, -5 }, /* (30) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ - { 160, -3 }, /* (31) cmd ::= DROP DNODE NK_INTEGER */ - { 160, -3 }, /* (32) cmd ::= DROP DNODE dnode_endpoint */ - { 160, -2 }, /* (33) cmd ::= SHOW DNODES */ - { 160, -4 }, /* (34) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 160, -5 }, /* (35) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 160, -4 }, /* (36) cmd ::= ALTER ALL DNODES NK_STRING */ - { 160, -5 }, /* (37) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 166, -1 }, /* (38) dnode_endpoint ::= NK_STRING */ - { 167, -1 }, /* (39) dnode_host_name ::= NK_ID */ - { 167, -1 }, /* (40) dnode_host_name ::= NK_IPTOKEN */ - { 160, -3 }, /* (41) cmd ::= ALTER LOCAL NK_STRING */ - { 160, -4 }, /* (42) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 160, -5 }, /* (43) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 160, -5 }, /* (44) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 160, -2 }, /* (45) cmd ::= SHOW QNODES */ - { 160, -5 }, /* (46) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 160, -4 }, /* (47) cmd ::= DROP DATABASE exists_opt db_name */ - { 160, -2 }, /* (48) cmd ::= SHOW DATABASES */ - { 160, -2 }, /* (49) cmd ::= USE db_name */ - { 160, -4 }, /* (50) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 168, -3 }, /* (51) not_exists_opt ::= IF NOT EXISTS */ - { 168, 0 }, /* (52) not_exists_opt ::= */ - { 171, -2 }, /* (53) exists_opt ::= IF EXISTS */ - { 171, 0 }, /* (54) exists_opt ::= */ - { 170, 0 }, /* (55) db_options ::= */ - { 170, -3 }, /* (56) db_options ::= db_options BLOCKS NK_INTEGER */ - { 170, -3 }, /* (57) db_options ::= db_options CACHE NK_INTEGER */ - { 170, -3 }, /* (58) db_options ::= db_options CACHELAST NK_INTEGER */ - { 170, -3 }, /* (59) db_options ::= db_options COMP NK_INTEGER */ - { 170, -3 }, /* (60) db_options ::= db_options DAYS NK_INTEGER */ - { 170, -3 }, /* (61) db_options ::= db_options FSYNC NK_INTEGER */ - { 170, -3 }, /* (62) db_options ::= db_options MAXROWS NK_INTEGER */ - { 170, -3 }, /* (63) db_options ::= db_options MINROWS NK_INTEGER */ - { 170, -3 }, /* (64) db_options ::= db_options KEEP NK_INTEGER */ - { 170, -3 }, /* (65) db_options ::= db_options PRECISION NK_STRING */ - { 170, -3 }, /* (66) db_options ::= db_options QUORUM NK_INTEGER */ - { 170, -3 }, /* (67) db_options ::= db_options REPLICA NK_INTEGER */ - { 170, -3 }, /* (68) db_options ::= db_options TTL NK_INTEGER */ - { 170, -3 }, /* (69) db_options ::= db_options WAL NK_INTEGER */ - { 170, -3 }, /* (70) db_options ::= db_options VGROUPS NK_INTEGER */ - { 170, -3 }, /* (71) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 170, -3 }, /* (72) db_options ::= db_options STREAM_MODE NK_INTEGER */ - { 170, -3 }, /* (73) db_options ::= db_options RETENTIONS NK_STRING */ - { 170, -3 }, /* (74) db_options ::= db_options FILE_FACTOR NK_FLOAT */ - { 172, -1 }, /* (75) alter_db_options ::= alter_db_option */ - { 172, -2 }, /* (76) alter_db_options ::= alter_db_options alter_db_option */ - { 173, -2 }, /* (77) alter_db_option ::= BLOCKS NK_INTEGER */ - { 173, -2 }, /* (78) alter_db_option ::= FSYNC NK_INTEGER */ - { 173, -2 }, /* (79) alter_db_option ::= KEEP NK_INTEGER */ - { 173, -2 }, /* (80) alter_db_option ::= WAL NK_INTEGER */ - { 173, -2 }, /* (81) alter_db_option ::= QUORUM NK_INTEGER */ - { 173, -2 }, /* (82) alter_db_option ::= CACHELAST NK_INTEGER */ - { 160, -9 }, /* (83) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 160, -3 }, /* (84) cmd ::= CREATE TABLE multi_create_clause */ - { 160, -9 }, /* (85) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 160, -3 }, /* (86) cmd ::= DROP TABLE multi_drop_clause */ - { 160, -4 }, /* (87) cmd ::= DROP STABLE exists_opt full_table_name */ - { 160, -2 }, /* (88) cmd ::= SHOW TABLES */ - { 160, -2 }, /* (89) cmd ::= SHOW STABLES */ - { 160, -3 }, /* (90) cmd ::= ALTER TABLE alter_table_clause */ - { 160, -3 }, /* (91) cmd ::= ALTER STABLE alter_table_clause */ - { 181, -2 }, /* (92) alter_table_clause ::= full_table_name alter_table_options */ - { 181, -5 }, /* (93) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 181, -4 }, /* (94) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 181, -5 }, /* (95) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 181, -5 }, /* (96) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 181, -5 }, /* (97) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 181, -4 }, /* (98) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 181, -5 }, /* (99) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 181, -5 }, /* (100) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 181, -6 }, /* (101) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ - { 178, -1 }, /* (102) multi_create_clause ::= create_subtable_clause */ - { 178, -2 }, /* (103) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 185, -9 }, /* (104) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ - { 180, -1 }, /* (105) multi_drop_clause ::= drop_table_clause */ - { 180, -2 }, /* (106) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 188, -2 }, /* (107) drop_table_clause ::= exists_opt full_table_name */ - { 186, 0 }, /* (108) specific_tags_opt ::= */ - { 186, -3 }, /* (109) specific_tags_opt ::= NK_LP col_name_list NK_RP */ - { 174, -1 }, /* (110) full_table_name ::= table_name */ - { 174, -3 }, /* (111) full_table_name ::= db_name NK_DOT table_name */ - { 175, -1 }, /* (112) column_def_list ::= column_def */ - { 175, -3 }, /* (113) column_def_list ::= column_def_list NK_COMMA column_def */ - { 191, -2 }, /* (114) column_def ::= column_name type_name */ - { 191, -4 }, /* (115) column_def ::= column_name type_name COMMENT NK_STRING */ - { 184, -1 }, /* (116) type_name ::= BOOL */ - { 184, -1 }, /* (117) type_name ::= TINYINT */ - { 184, -1 }, /* (118) type_name ::= SMALLINT */ - { 184, -1 }, /* (119) type_name ::= INT */ - { 184, -1 }, /* (120) type_name ::= INTEGER */ - { 184, -1 }, /* (121) type_name ::= BIGINT */ - { 184, -1 }, /* (122) type_name ::= FLOAT */ - { 184, -1 }, /* (123) type_name ::= DOUBLE */ - { 184, -4 }, /* (124) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 184, -1 }, /* (125) type_name ::= TIMESTAMP */ - { 184, -4 }, /* (126) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 184, -2 }, /* (127) type_name ::= TINYINT UNSIGNED */ - { 184, -2 }, /* (128) type_name ::= SMALLINT UNSIGNED */ - { 184, -2 }, /* (129) type_name ::= INT UNSIGNED */ - { 184, -2 }, /* (130) type_name ::= BIGINT UNSIGNED */ - { 184, -1 }, /* (131) type_name ::= JSON */ - { 184, -4 }, /* (132) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 184, -1 }, /* (133) type_name ::= MEDIUMBLOB */ - { 184, -1 }, /* (134) type_name ::= BLOB */ - { 184, -4 }, /* (135) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 184, -1 }, /* (136) type_name ::= DECIMAL */ - { 184, -4 }, /* (137) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 184, -6 }, /* (138) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 176, 0 }, /* (139) tags_def_opt ::= */ - { 176, -1 }, /* (140) tags_def_opt ::= tags_def */ - { 179, -4 }, /* (141) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 177, 0 }, /* (142) table_options ::= */ - { 177, -3 }, /* (143) table_options ::= table_options COMMENT NK_STRING */ - { 177, -3 }, /* (144) table_options ::= table_options KEEP NK_INTEGER */ - { 177, -3 }, /* (145) table_options ::= table_options TTL NK_INTEGER */ - { 177, -5 }, /* (146) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 177, -5 }, /* (147) table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ - { 182, -1 }, /* (148) alter_table_options ::= alter_table_option */ - { 182, -2 }, /* (149) alter_table_options ::= alter_table_options alter_table_option */ - { 193, -2 }, /* (150) alter_table_option ::= COMMENT NK_STRING */ - { 193, -2 }, /* (151) alter_table_option ::= KEEP NK_INTEGER */ - { 193, -2 }, /* (152) alter_table_option ::= TTL NK_INTEGER */ - { 189, -1 }, /* (153) col_name_list ::= col_name */ - { 189, -3 }, /* (154) col_name_list ::= col_name_list NK_COMMA col_name */ - { 194, -1 }, /* (155) col_name ::= column_name */ - { 192, -1 }, /* (156) func_name_list ::= func_name */ - { 192, -3 }, /* (157) func_name_list ::= func_name_list NK_COMMA col_name */ - { 195, -1 }, /* (158) func_name ::= function_name */ - { 160, -7 }, /* (159) cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ - { 160, -9 }, /* (160) cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ - { 160, -5 }, /* (161) cmd ::= DROP INDEX index_name ON table_name */ - { 198, 0 }, /* (162) index_options ::= */ - { 198, -9 }, /* (163) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ - { 198, -11 }, /* (164) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ - { 199, -1 }, /* (165) func_list ::= func */ - { 199, -3 }, /* (166) func_list ::= func_list NK_COMMA func */ - { 202, -4 }, /* (167) func ::= function_name NK_LP expression_list NK_RP */ - { 160, -6 }, /* (168) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ - { 160, -6 }, /* (169) cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ - { 160, -4 }, /* (170) cmd ::= DROP TOPIC exists_opt topic_name */ - { 160, -2 }, /* (171) cmd ::= SHOW VGROUPS */ - { 160, -4 }, /* (172) cmd ::= SHOW db_name NK_DOT VGROUPS */ - { 160, -2 }, /* (173) cmd ::= SHOW MNODES */ - { 160, -1 }, /* (174) cmd ::= query_expression */ - { 163, -1 }, /* (175) literal ::= NK_INTEGER */ - { 163, -1 }, /* (176) literal ::= NK_FLOAT */ - { 163, -1 }, /* (177) literal ::= NK_STRING */ - { 163, -1 }, /* (178) literal ::= NK_BOOL */ - { 163, -2 }, /* (179) literal ::= TIMESTAMP NK_STRING */ - { 163, -1 }, /* (180) literal ::= duration_literal */ - { 200, -1 }, /* (181) duration_literal ::= NK_VARIABLE */ - { 187, -1 }, /* (182) literal_list ::= literal */ - { 187, -3 }, /* (183) literal_list ::= literal_list NK_COMMA literal */ - { 169, -1 }, /* (184) db_name ::= NK_ID */ - { 190, -1 }, /* (185) table_name ::= NK_ID */ - { 183, -1 }, /* (186) column_name ::= NK_ID */ - { 196, -1 }, /* (187) function_name ::= NK_ID */ - { 206, -1 }, /* (188) table_alias ::= NK_ID */ - { 207, -1 }, /* (189) column_alias ::= NK_ID */ - { 165, -1 }, /* (190) user_name ::= NK_ID */ - { 197, -1 }, /* (191) index_name ::= NK_ID */ - { 204, -1 }, /* (192) topic_name ::= NK_ID */ - { 208, -1 }, /* (193) expression ::= literal */ - { 208, -1 }, /* (194) expression ::= column_reference */ - { 208, -4 }, /* (195) expression ::= function_name NK_LP expression_list NK_RP */ - { 208, -4 }, /* (196) expression ::= function_name NK_LP NK_STAR NK_RP */ - { 208, -1 }, /* (197) expression ::= subquery */ - { 208, -3 }, /* (198) expression ::= NK_LP expression NK_RP */ - { 208, -2 }, /* (199) expression ::= NK_PLUS expression */ - { 208, -2 }, /* (200) expression ::= NK_MINUS expression */ - { 208, -3 }, /* (201) expression ::= expression NK_PLUS expression */ - { 208, -3 }, /* (202) expression ::= expression NK_MINUS expression */ - { 208, -3 }, /* (203) expression ::= expression NK_STAR expression */ - { 208, -3 }, /* (204) expression ::= expression NK_SLASH expression */ - { 208, -3 }, /* (205) expression ::= expression NK_REM expression */ - { 203, -1 }, /* (206) expression_list ::= expression */ - { 203, -3 }, /* (207) expression_list ::= expression_list NK_COMMA expression */ - { 209, -1 }, /* (208) column_reference ::= column_name */ - { 209, -3 }, /* (209) column_reference ::= table_name NK_DOT column_name */ - { 211, -3 }, /* (210) predicate ::= expression compare_op expression */ - { 211, -5 }, /* (211) predicate ::= expression BETWEEN expression AND expression */ - { 211, -6 }, /* (212) predicate ::= expression NOT BETWEEN expression AND expression */ - { 211, -3 }, /* (213) predicate ::= expression IS NULL */ - { 211, -4 }, /* (214) predicate ::= expression IS NOT NULL */ - { 211, -3 }, /* (215) predicate ::= expression in_op in_predicate_value */ - { 212, -1 }, /* (216) compare_op ::= NK_LT */ - { 212, -1 }, /* (217) compare_op ::= NK_GT */ - { 212, -1 }, /* (218) compare_op ::= NK_LE */ - { 212, -1 }, /* (219) compare_op ::= NK_GE */ - { 212, -1 }, /* (220) compare_op ::= NK_NE */ - { 212, -1 }, /* (221) compare_op ::= NK_EQ */ - { 212, -1 }, /* (222) compare_op ::= LIKE */ - { 212, -2 }, /* (223) compare_op ::= NOT LIKE */ - { 212, -1 }, /* (224) compare_op ::= MATCH */ - { 212, -1 }, /* (225) compare_op ::= NMATCH */ - { 213, -1 }, /* (226) in_op ::= IN */ - { 213, -2 }, /* (227) in_op ::= NOT IN */ - { 214, -3 }, /* (228) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 215, -1 }, /* (229) boolean_value_expression ::= boolean_primary */ - { 215, -2 }, /* (230) boolean_value_expression ::= NOT boolean_primary */ - { 215, -3 }, /* (231) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 215, -3 }, /* (232) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 216, -1 }, /* (233) boolean_primary ::= predicate */ - { 216, -3 }, /* (234) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 217, -1 }, /* (235) common_expression ::= expression */ - { 217, -1 }, /* (236) common_expression ::= boolean_value_expression */ - { 218, -2 }, /* (237) from_clause ::= FROM table_reference_list */ - { 219, -1 }, /* (238) table_reference_list ::= table_reference */ - { 219, -3 }, /* (239) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 220, -1 }, /* (240) table_reference ::= table_primary */ - { 220, -1 }, /* (241) table_reference ::= joined_table */ - { 221, -2 }, /* (242) table_primary ::= table_name alias_opt */ - { 221, -4 }, /* (243) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 221, -2 }, /* (244) table_primary ::= subquery alias_opt */ - { 221, -1 }, /* (245) table_primary ::= parenthesized_joined_table */ - { 223, 0 }, /* (246) alias_opt ::= */ - { 223, -1 }, /* (247) alias_opt ::= table_alias */ - { 223, -2 }, /* (248) alias_opt ::= AS table_alias */ - { 224, -3 }, /* (249) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 224, -3 }, /* (250) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 222, -6 }, /* (251) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 225, 0 }, /* (252) join_type ::= */ - { 225, -1 }, /* (253) join_type ::= INNER */ - { 227, -9 }, /* (254) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 228, 0 }, /* (255) set_quantifier_opt ::= */ - { 228, -1 }, /* (256) set_quantifier_opt ::= DISTINCT */ - { 228, -1 }, /* (257) set_quantifier_opt ::= ALL */ - { 229, -1 }, /* (258) select_list ::= NK_STAR */ - { 229, -1 }, /* (259) select_list ::= select_sublist */ - { 235, -1 }, /* (260) select_sublist ::= select_item */ - { 235, -3 }, /* (261) select_sublist ::= select_sublist NK_COMMA select_item */ - { 236, -1 }, /* (262) select_item ::= common_expression */ - { 236, -2 }, /* (263) select_item ::= common_expression column_alias */ - { 236, -3 }, /* (264) select_item ::= common_expression AS column_alias */ - { 236, -3 }, /* (265) select_item ::= table_name NK_DOT NK_STAR */ - { 230, 0 }, /* (266) where_clause_opt ::= */ - { 230, -2 }, /* (267) where_clause_opt ::= WHERE search_condition */ - { 231, 0 }, /* (268) partition_by_clause_opt ::= */ - { 231, -3 }, /* (269) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 232, 0 }, /* (270) twindow_clause_opt ::= */ - { 232, -6 }, /* (271) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ - { 232, -4 }, /* (272) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ - { 232, -6 }, /* (273) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 232, -8 }, /* (274) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 201, 0 }, /* (275) sliding_opt ::= */ - { 201, -4 }, /* (276) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 237, 0 }, /* (277) fill_opt ::= */ - { 237, -4 }, /* (278) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 237, -6 }, /* (279) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 238, -1 }, /* (280) fill_mode ::= NONE */ - { 238, -1 }, /* (281) fill_mode ::= PREV */ - { 238, -1 }, /* (282) fill_mode ::= NULL */ - { 238, -1 }, /* (283) fill_mode ::= LINEAR */ - { 238, -1 }, /* (284) fill_mode ::= NEXT */ - { 233, 0 }, /* (285) group_by_clause_opt ::= */ - { 233, -3 }, /* (286) group_by_clause_opt ::= GROUP BY group_by_list */ - { 239, -1 }, /* (287) group_by_list ::= expression */ - { 239, -3 }, /* (288) group_by_list ::= group_by_list NK_COMMA expression */ - { 234, 0 }, /* (289) having_clause_opt ::= */ - { 234, -2 }, /* (290) having_clause_opt ::= HAVING search_condition */ - { 205, -4 }, /* (291) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 240, -1 }, /* (292) query_expression_body ::= query_primary */ - { 240, -4 }, /* (293) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 244, -1 }, /* (294) query_primary ::= query_specification */ - { 241, 0 }, /* (295) order_by_clause_opt ::= */ - { 241, -3 }, /* (296) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 242, 0 }, /* (297) slimit_clause_opt ::= */ - { 242, -2 }, /* (298) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 242, -4 }, /* (299) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 242, -4 }, /* (300) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 243, 0 }, /* (301) limit_clause_opt ::= */ - { 243, -2 }, /* (302) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 243, -4 }, /* (303) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 243, -4 }, /* (304) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 210, -3 }, /* (305) subquery ::= NK_LP query_expression NK_RP */ - { 226, -1 }, /* (306) search_condition ::= common_expression */ - { 245, -1 }, /* (307) sort_specification_list ::= sort_specification */ - { 245, -3 }, /* (308) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 246, -3 }, /* (309) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 247, 0 }, /* (310) ordering_specification_opt ::= */ - { 247, -1 }, /* (311) ordering_specification_opt ::= ASC */ - { 247, -1 }, /* (312) ordering_specification_opt ::= DESC */ - { 248, 0 }, /* (313) null_ordering_opt ::= */ - { 248, -2 }, /* (314) null_ordering_opt ::= NULLS FIRST */ - { 248, -2 }, /* (315) null_ordering_opt ::= NULLS LAST */ + { 163, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 163, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 164, 0 }, /* (2) account_options ::= */ + { 164, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 164, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 164, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 164, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 164, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 164, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 164, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 164, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 164, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 165, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 165, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 167, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 167, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 167, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 167, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 167, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 167, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 167, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 167, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 167, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 167, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 163, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ + { 163, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 163, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ + { 163, -3 }, /* (27) cmd ::= DROP USER user_name */ + { 163, -3 }, /* (28) cmd ::= CREATE DNODE dnode_endpoint */ + { 163, -5 }, /* (29) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ + { 163, -3 }, /* (30) cmd ::= DROP DNODE NK_INTEGER */ + { 163, -3 }, /* (31) cmd ::= DROP DNODE dnode_endpoint */ + { 163, -4 }, /* (32) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 163, -5 }, /* (33) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 163, -4 }, /* (34) cmd ::= ALTER ALL DNODES NK_STRING */ + { 163, -5 }, /* (35) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 169, -1 }, /* (36) dnode_endpoint ::= NK_STRING */ + { 170, -1 }, /* (37) dnode_host_name ::= NK_ID */ + { 170, -1 }, /* (38) dnode_host_name ::= NK_IPTOKEN */ + { 163, -3 }, /* (39) cmd ::= ALTER LOCAL NK_STRING */ + { 163, -4 }, /* (40) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 163, -5 }, /* (41) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 163, -5 }, /* (42) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 163, -5 }, /* (43) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 163, -4 }, /* (44) cmd ::= DROP DATABASE exists_opt db_name */ + { 163, -2 }, /* (45) cmd ::= USE db_name */ + { 163, -4 }, /* (46) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 171, -3 }, /* (47) not_exists_opt ::= IF NOT EXISTS */ + { 171, 0 }, /* (48) not_exists_opt ::= */ + { 174, -2 }, /* (49) exists_opt ::= IF EXISTS */ + { 174, 0 }, /* (50) exists_opt ::= */ + { 173, 0 }, /* (51) db_options ::= */ + { 173, -3 }, /* (52) db_options ::= db_options BLOCKS NK_INTEGER */ + { 173, -3 }, /* (53) db_options ::= db_options CACHE NK_INTEGER */ + { 173, -3 }, /* (54) db_options ::= db_options CACHELAST NK_INTEGER */ + { 173, -3 }, /* (55) db_options ::= db_options COMP NK_INTEGER */ + { 173, -3 }, /* (56) db_options ::= db_options DAYS NK_INTEGER */ + { 173, -3 }, /* (57) db_options ::= db_options FSYNC NK_INTEGER */ + { 173, -3 }, /* (58) db_options ::= db_options MAXROWS NK_INTEGER */ + { 173, -3 }, /* (59) db_options ::= db_options MINROWS NK_INTEGER */ + { 173, -3 }, /* (60) db_options ::= db_options KEEP NK_INTEGER */ + { 173, -3 }, /* (61) db_options ::= db_options PRECISION NK_STRING */ + { 173, -3 }, /* (62) db_options ::= db_options QUORUM NK_INTEGER */ + { 173, -3 }, /* (63) db_options ::= db_options REPLICA NK_INTEGER */ + { 173, -3 }, /* (64) db_options ::= db_options TTL NK_INTEGER */ + { 173, -3 }, /* (65) db_options ::= db_options WAL NK_INTEGER */ + { 173, -3 }, /* (66) db_options ::= db_options VGROUPS NK_INTEGER */ + { 173, -3 }, /* (67) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 173, -3 }, /* (68) db_options ::= db_options STREAM_MODE NK_INTEGER */ + { 173, -3 }, /* (69) db_options ::= db_options RETENTIONS NK_STRING */ + { 173, -3 }, /* (70) db_options ::= db_options FILE_FACTOR NK_FLOAT */ + { 175, -1 }, /* (71) alter_db_options ::= alter_db_option */ + { 175, -2 }, /* (72) alter_db_options ::= alter_db_options alter_db_option */ + { 176, -2 }, /* (73) alter_db_option ::= BLOCKS NK_INTEGER */ + { 176, -2 }, /* (74) alter_db_option ::= FSYNC NK_INTEGER */ + { 176, -2 }, /* (75) alter_db_option ::= KEEP NK_INTEGER */ + { 176, -2 }, /* (76) alter_db_option ::= WAL NK_INTEGER */ + { 176, -2 }, /* (77) alter_db_option ::= QUORUM NK_INTEGER */ + { 176, -2 }, /* (78) alter_db_option ::= CACHELAST NK_INTEGER */ + { 163, -9 }, /* (79) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 163, -3 }, /* (80) cmd ::= CREATE TABLE multi_create_clause */ + { 163, -9 }, /* (81) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 163, -3 }, /* (82) cmd ::= DROP TABLE multi_drop_clause */ + { 163, -4 }, /* (83) cmd ::= DROP STABLE exists_opt full_table_name */ + { 163, -3 }, /* (84) cmd ::= ALTER TABLE alter_table_clause */ + { 163, -3 }, /* (85) cmd ::= ALTER STABLE alter_table_clause */ + { 184, -2 }, /* (86) alter_table_clause ::= full_table_name alter_table_options */ + { 184, -5 }, /* (87) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 184, -4 }, /* (88) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 184, -5 }, /* (89) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 184, -5 }, /* (90) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 184, -5 }, /* (91) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 184, -4 }, /* (92) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 184, -5 }, /* (93) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 184, -5 }, /* (94) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 184, -6 }, /* (95) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ + { 181, -1 }, /* (96) multi_create_clause ::= create_subtable_clause */ + { 181, -2 }, /* (97) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 188, -9 }, /* (98) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ + { 183, -1 }, /* (99) multi_drop_clause ::= drop_table_clause */ + { 183, -2 }, /* (100) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 191, -2 }, /* (101) drop_table_clause ::= exists_opt full_table_name */ + { 189, 0 }, /* (102) specific_tags_opt ::= */ + { 189, -3 }, /* (103) specific_tags_opt ::= NK_LP col_name_list NK_RP */ + { 177, -1 }, /* (104) full_table_name ::= table_name */ + { 177, -3 }, /* (105) full_table_name ::= db_name NK_DOT table_name */ + { 178, -1 }, /* (106) column_def_list ::= column_def */ + { 178, -3 }, /* (107) column_def_list ::= column_def_list NK_COMMA column_def */ + { 194, -2 }, /* (108) column_def ::= column_name type_name */ + { 194, -4 }, /* (109) column_def ::= column_name type_name COMMENT NK_STRING */ + { 187, -1 }, /* (110) type_name ::= BOOL */ + { 187, -1 }, /* (111) type_name ::= TINYINT */ + { 187, -1 }, /* (112) type_name ::= SMALLINT */ + { 187, -1 }, /* (113) type_name ::= INT */ + { 187, -1 }, /* (114) type_name ::= INTEGER */ + { 187, -1 }, /* (115) type_name ::= BIGINT */ + { 187, -1 }, /* (116) type_name ::= FLOAT */ + { 187, -1 }, /* (117) type_name ::= DOUBLE */ + { 187, -4 }, /* (118) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 187, -1 }, /* (119) type_name ::= TIMESTAMP */ + { 187, -4 }, /* (120) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 187, -2 }, /* (121) type_name ::= TINYINT UNSIGNED */ + { 187, -2 }, /* (122) type_name ::= SMALLINT UNSIGNED */ + { 187, -2 }, /* (123) type_name ::= INT UNSIGNED */ + { 187, -2 }, /* (124) type_name ::= BIGINT UNSIGNED */ + { 187, -1 }, /* (125) type_name ::= JSON */ + { 187, -4 }, /* (126) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 187, -1 }, /* (127) type_name ::= MEDIUMBLOB */ + { 187, -1 }, /* (128) type_name ::= BLOB */ + { 187, -4 }, /* (129) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 187, -1 }, /* (130) type_name ::= DECIMAL */ + { 187, -4 }, /* (131) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 187, -6 }, /* (132) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 179, 0 }, /* (133) tags_def_opt ::= */ + { 179, -1 }, /* (134) tags_def_opt ::= tags_def */ + { 182, -4 }, /* (135) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 180, 0 }, /* (136) table_options ::= */ + { 180, -3 }, /* (137) table_options ::= table_options COMMENT NK_STRING */ + { 180, -3 }, /* (138) table_options ::= table_options KEEP NK_INTEGER */ + { 180, -3 }, /* (139) table_options ::= table_options TTL NK_INTEGER */ + { 180, -5 }, /* (140) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 180, -5 }, /* (141) table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ + { 185, -1 }, /* (142) alter_table_options ::= alter_table_option */ + { 185, -2 }, /* (143) alter_table_options ::= alter_table_options alter_table_option */ + { 196, -2 }, /* (144) alter_table_option ::= COMMENT NK_STRING */ + { 196, -2 }, /* (145) alter_table_option ::= KEEP NK_INTEGER */ + { 196, -2 }, /* (146) alter_table_option ::= TTL NK_INTEGER */ + { 192, -1 }, /* (147) col_name_list ::= col_name */ + { 192, -3 }, /* (148) col_name_list ::= col_name_list NK_COMMA col_name */ + { 197, -1 }, /* (149) col_name ::= column_name */ + { 163, -2 }, /* (150) cmd ::= SHOW DNODES */ + { 163, -2 }, /* (151) cmd ::= SHOW USERS */ + { 163, -2 }, /* (152) cmd ::= SHOW DATABASES */ + { 163, -4 }, /* (153) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 163, -4 }, /* (154) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 163, -3 }, /* (155) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 163, -2 }, /* (156) cmd ::= SHOW MNODES */ + { 163, -2 }, /* (157) cmd ::= SHOW MODULES */ + { 163, -2 }, /* (158) cmd ::= SHOW QNODES */ + { 163, -2 }, /* (159) cmd ::= SHOW FUNCTIONS */ + { 163, -5 }, /* (160) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 163, -2 }, /* (161) cmd ::= SHOW STREAMS */ + { 198, 0 }, /* (162) db_name_cond_opt ::= */ + { 198, -2 }, /* (163) db_name_cond_opt ::= db_name NK_DOT */ + { 199, 0 }, /* (164) like_pattern_opt ::= */ + { 199, -2 }, /* (165) like_pattern_opt ::= LIKE NK_STRING */ + { 200, -1 }, /* (166) table_name_cond ::= table_name */ + { 201, 0 }, /* (167) from_db_opt ::= */ + { 201, -2 }, /* (168) from_db_opt ::= FROM db_name */ + { 195, -1 }, /* (169) func_name_list ::= func_name */ + { 195, -3 }, /* (170) func_name_list ::= func_name_list NK_COMMA col_name */ + { 202, -1 }, /* (171) func_name ::= function_name */ + { 163, -8 }, /* (172) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ + { 163, -10 }, /* (173) cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ + { 163, -6 }, /* (174) cmd ::= DROP INDEX exists_opt index_name ON table_name */ + { 205, 0 }, /* (175) index_options ::= */ + { 205, -9 }, /* (176) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ + { 205, -11 }, /* (177) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ + { 206, -1 }, /* (178) func_list ::= func */ + { 206, -3 }, /* (179) func_list ::= func_list NK_COMMA func */ + { 209, -4 }, /* (180) func ::= function_name NK_LP expression_list NK_RP */ + { 163, -6 }, /* (181) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ + { 163, -6 }, /* (182) cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ + { 163, -4 }, /* (183) cmd ::= DROP TOPIC exists_opt topic_name */ + { 163, -1 }, /* (184) cmd ::= query_expression */ + { 166, -1 }, /* (185) literal ::= NK_INTEGER */ + { 166, -1 }, /* (186) literal ::= NK_FLOAT */ + { 166, -1 }, /* (187) literal ::= NK_STRING */ + { 166, -1 }, /* (188) literal ::= NK_BOOL */ + { 166, -2 }, /* (189) literal ::= TIMESTAMP NK_STRING */ + { 166, -1 }, /* (190) literal ::= duration_literal */ + { 207, -1 }, /* (191) duration_literal ::= NK_VARIABLE */ + { 213, -1 }, /* (192) signed ::= NK_INTEGER */ + { 213, -2 }, /* (193) signed ::= NK_PLUS NK_INTEGER */ + { 213, -2 }, /* (194) signed ::= NK_MINUS NK_INTEGER */ + { 213, -1 }, /* (195) signed ::= NK_FLOAT */ + { 213, -2 }, /* (196) signed ::= NK_PLUS NK_FLOAT */ + { 213, -2 }, /* (197) signed ::= NK_MINUS NK_FLOAT */ + { 214, -1 }, /* (198) signed_literal ::= signed */ + { 214, -1 }, /* (199) signed_literal ::= NK_STRING */ + { 214, -1 }, /* (200) signed_literal ::= NK_BOOL */ + { 214, -2 }, /* (201) signed_literal ::= TIMESTAMP NK_STRING */ + { 214, -1 }, /* (202) signed_literal ::= duration_literal */ + { 190, -1 }, /* (203) literal_list ::= signed_literal */ + { 190, -3 }, /* (204) literal_list ::= literal_list NK_COMMA signed_literal */ + { 172, -1 }, /* (205) db_name ::= NK_ID */ + { 193, -1 }, /* (206) table_name ::= NK_ID */ + { 186, -1 }, /* (207) column_name ::= NK_ID */ + { 203, -1 }, /* (208) function_name ::= NK_ID */ + { 215, -1 }, /* (209) table_alias ::= NK_ID */ + { 216, -1 }, /* (210) column_alias ::= NK_ID */ + { 168, -1 }, /* (211) user_name ::= NK_ID */ + { 204, -1 }, /* (212) index_name ::= NK_ID */ + { 211, -1 }, /* (213) topic_name ::= NK_ID */ + { 217, -1 }, /* (214) expression ::= literal */ + { 217, -1 }, /* (215) expression ::= column_reference */ + { 217, -4 }, /* (216) expression ::= function_name NK_LP expression_list NK_RP */ + { 217, -4 }, /* (217) expression ::= function_name NK_LP NK_STAR NK_RP */ + { 217, -1 }, /* (218) expression ::= subquery */ + { 217, -3 }, /* (219) expression ::= NK_LP expression NK_RP */ + { 217, -2 }, /* (220) expression ::= NK_PLUS expression */ + { 217, -2 }, /* (221) expression ::= NK_MINUS expression */ + { 217, -3 }, /* (222) expression ::= expression NK_PLUS expression */ + { 217, -3 }, /* (223) expression ::= expression NK_MINUS expression */ + { 217, -3 }, /* (224) expression ::= expression NK_STAR expression */ + { 217, -3 }, /* (225) expression ::= expression NK_SLASH expression */ + { 217, -3 }, /* (226) expression ::= expression NK_REM expression */ + { 210, -1 }, /* (227) expression_list ::= expression */ + { 210, -3 }, /* (228) expression_list ::= expression_list NK_COMMA expression */ + { 218, -1 }, /* (229) column_reference ::= column_name */ + { 218, -3 }, /* (230) column_reference ::= table_name NK_DOT column_name */ + { 220, -3 }, /* (231) predicate ::= expression compare_op expression */ + { 220, -5 }, /* (232) predicate ::= expression BETWEEN expression AND expression */ + { 220, -6 }, /* (233) predicate ::= expression NOT BETWEEN expression AND expression */ + { 220, -3 }, /* (234) predicate ::= expression IS NULL */ + { 220, -4 }, /* (235) predicate ::= expression IS NOT NULL */ + { 220, -3 }, /* (236) predicate ::= expression in_op in_predicate_value */ + { 221, -1 }, /* (237) compare_op ::= NK_LT */ + { 221, -1 }, /* (238) compare_op ::= NK_GT */ + { 221, -1 }, /* (239) compare_op ::= NK_LE */ + { 221, -1 }, /* (240) compare_op ::= NK_GE */ + { 221, -1 }, /* (241) compare_op ::= NK_NE */ + { 221, -1 }, /* (242) compare_op ::= NK_EQ */ + { 221, -1 }, /* (243) compare_op ::= LIKE */ + { 221, -2 }, /* (244) compare_op ::= NOT LIKE */ + { 221, -1 }, /* (245) compare_op ::= MATCH */ + { 221, -1 }, /* (246) compare_op ::= NMATCH */ + { 222, -1 }, /* (247) in_op ::= IN */ + { 222, -2 }, /* (248) in_op ::= NOT IN */ + { 223, -3 }, /* (249) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 224, -1 }, /* (250) boolean_value_expression ::= boolean_primary */ + { 224, -2 }, /* (251) boolean_value_expression ::= NOT boolean_primary */ + { 224, -3 }, /* (252) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 224, -3 }, /* (253) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 225, -1 }, /* (254) boolean_primary ::= predicate */ + { 225, -3 }, /* (255) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 226, -1 }, /* (256) common_expression ::= expression */ + { 226, -1 }, /* (257) common_expression ::= boolean_value_expression */ + { 227, -2 }, /* (258) from_clause ::= FROM table_reference_list */ + { 228, -1 }, /* (259) table_reference_list ::= table_reference */ + { 228, -3 }, /* (260) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 229, -1 }, /* (261) table_reference ::= table_primary */ + { 229, -1 }, /* (262) table_reference ::= joined_table */ + { 230, -2 }, /* (263) table_primary ::= table_name alias_opt */ + { 230, -4 }, /* (264) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 230, -2 }, /* (265) table_primary ::= subquery alias_opt */ + { 230, -1 }, /* (266) table_primary ::= parenthesized_joined_table */ + { 232, 0 }, /* (267) alias_opt ::= */ + { 232, -1 }, /* (268) alias_opt ::= table_alias */ + { 232, -2 }, /* (269) alias_opt ::= AS table_alias */ + { 233, -3 }, /* (270) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 233, -3 }, /* (271) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 231, -6 }, /* (272) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 234, 0 }, /* (273) join_type ::= */ + { 234, -1 }, /* (274) join_type ::= INNER */ + { 236, -9 }, /* (275) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 237, 0 }, /* (276) set_quantifier_opt ::= */ + { 237, -1 }, /* (277) set_quantifier_opt ::= DISTINCT */ + { 237, -1 }, /* (278) set_quantifier_opt ::= ALL */ + { 238, -1 }, /* (279) select_list ::= NK_STAR */ + { 238, -1 }, /* (280) select_list ::= select_sublist */ + { 244, -1 }, /* (281) select_sublist ::= select_item */ + { 244, -3 }, /* (282) select_sublist ::= select_sublist NK_COMMA select_item */ + { 245, -1 }, /* (283) select_item ::= common_expression */ + { 245, -2 }, /* (284) select_item ::= common_expression column_alias */ + { 245, -3 }, /* (285) select_item ::= common_expression AS column_alias */ + { 245, -3 }, /* (286) select_item ::= table_name NK_DOT NK_STAR */ + { 239, 0 }, /* (287) where_clause_opt ::= */ + { 239, -2 }, /* (288) where_clause_opt ::= WHERE search_condition */ + { 240, 0 }, /* (289) partition_by_clause_opt ::= */ + { 240, -3 }, /* (290) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 241, 0 }, /* (291) twindow_clause_opt ::= */ + { 241, -6 }, /* (292) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 241, -4 }, /* (293) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 241, -6 }, /* (294) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 241, -8 }, /* (295) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 208, 0 }, /* (296) sliding_opt ::= */ + { 208, -4 }, /* (297) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 246, 0 }, /* (298) fill_opt ::= */ + { 246, -4 }, /* (299) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 246, -6 }, /* (300) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 247, -1 }, /* (301) fill_mode ::= NONE */ + { 247, -1 }, /* (302) fill_mode ::= PREV */ + { 247, -1 }, /* (303) fill_mode ::= NULL */ + { 247, -1 }, /* (304) fill_mode ::= LINEAR */ + { 247, -1 }, /* (305) fill_mode ::= NEXT */ + { 242, 0 }, /* (306) group_by_clause_opt ::= */ + { 242, -3 }, /* (307) group_by_clause_opt ::= GROUP BY group_by_list */ + { 248, -1 }, /* (308) group_by_list ::= expression */ + { 248, -3 }, /* (309) group_by_list ::= group_by_list NK_COMMA expression */ + { 243, 0 }, /* (310) having_clause_opt ::= */ + { 243, -2 }, /* (311) having_clause_opt ::= HAVING search_condition */ + { 212, -4 }, /* (312) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 249, -1 }, /* (313) query_expression_body ::= query_primary */ + { 249, -4 }, /* (314) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 253, -1 }, /* (315) query_primary ::= query_specification */ + { 250, 0 }, /* (316) order_by_clause_opt ::= */ + { 250, -3 }, /* (317) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 251, 0 }, /* (318) slimit_clause_opt ::= */ + { 251, -2 }, /* (319) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 251, -4 }, /* (320) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 251, -4 }, /* (321) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 252, 0 }, /* (322) limit_clause_opt ::= */ + { 252, -2 }, /* (323) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 252, -4 }, /* (324) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 252, -4 }, /* (325) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 219, -3 }, /* (326) subquery ::= NK_LP query_expression NK_RP */ + { 235, -1 }, /* (327) search_condition ::= common_expression */ + { 254, -1 }, /* (328) sort_specification_list ::= sort_specification */ + { 254, -3 }, /* (329) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 255, -3 }, /* (330) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 256, 0 }, /* (331) ordering_specification_opt ::= */ + { 256, -1 }, /* (332) ordering_specification_opt ::= ASC */ + { 256, -1 }, /* (333) ordering_specification_opt ::= DESC */ + { 257, 0 }, /* (334) null_ordering_opt ::= */ + { 257, -2 }, /* (335) null_ordering_opt ::= NULLS FIRST */ + { 257, -2 }, /* (336) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2220,11 +2296,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,161,&yymsp[0].minor); + yy_destructor(yypParser,164,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,162,&yymsp[0].minor); + yy_destructor(yypParser,165,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -2238,20 +2314,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,161,&yymsp[-2].minor); +{ yy_destructor(yypParser,164,&yymsp[-2].minor); { } - yy_destructor(yypParser,163,&yymsp[0].minor); + yy_destructor(yypParser,166,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,164,&yymsp[0].minor); +{ yy_destructor(yypParser,167,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,162,&yymsp[-1].minor); +{ yy_destructor(yypParser,165,&yymsp[-1].minor); { } - yy_destructor(yypParser,164,&yymsp[0].minor); + yy_destructor(yypParser,167,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -2265,917 +2341,988 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,163,&yymsp[0].minor); + yy_destructor(yypParser,166,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy0); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy353, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy149, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy353, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy149, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy353); } - break; - case 28: /* cmd ::= SHOW USERS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy149); } break; - case 29: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy353, NULL); } + case 28: /* cmd ::= CREATE DNODE dnode_endpoint */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy149, NULL); } break; - case 30: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy0); } + case 29: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy0); } break; - case 31: /* cmd ::= DROP DNODE NK_INTEGER */ + case 30: /* cmd ::= DROP DNODE NK_INTEGER */ { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 32: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy353); } + case 31: /* cmd ::= DROP DNODE dnode_endpoint */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy149); } break; - case 33: /* cmd ::= SHOW DNODES */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } - break; - case 34: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + case 32: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; - case 35: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + case 33: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 36: /* cmd ::= ALTER ALL DNODES NK_STRING */ + case 34: /* cmd ::= ALTER ALL DNODES NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } break; - case 37: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + case 35: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 38: /* dnode_endpoint ::= NK_STRING */ - case 39: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==39); - case 40: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==40); - case 184: /* db_name ::= NK_ID */ yytestcase(yyruleno==184); - case 185: /* table_name ::= NK_ID */ yytestcase(yyruleno==185); - case 186: /* column_name ::= NK_ID */ yytestcase(yyruleno==186); - case 187: /* function_name ::= NK_ID */ yytestcase(yyruleno==187); - case 188: /* table_alias ::= NK_ID */ yytestcase(yyruleno==188); - case 189: /* column_alias ::= NK_ID */ yytestcase(yyruleno==189); - case 190: /* user_name ::= NK_ID */ yytestcase(yyruleno==190); - case 191: /* index_name ::= NK_ID */ yytestcase(yyruleno==191); - case 192: /* topic_name ::= NK_ID */ yytestcase(yyruleno==192); -{ yylhsminor.yy353 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy353 = yylhsminor.yy353; - break; - case 41: /* cmd ::= ALTER LOCAL NK_STRING */ + case 36: /* dnode_endpoint ::= NK_STRING */ + case 37: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==37); + case 38: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==38); + case 205: /* db_name ::= NK_ID */ yytestcase(yyruleno==205); + case 206: /* table_name ::= NK_ID */ yytestcase(yyruleno==206); + case 207: /* column_name ::= NK_ID */ yytestcase(yyruleno==207); + case 208: /* function_name ::= NK_ID */ yytestcase(yyruleno==208); + case 209: /* table_alias ::= NK_ID */ yytestcase(yyruleno==209); + case 210: /* column_alias ::= NK_ID */ yytestcase(yyruleno==210); + case 211: /* user_name ::= NK_ID */ yytestcase(yyruleno==211); + case 212: /* index_name ::= NK_ID */ yytestcase(yyruleno==212); + case 213: /* topic_name ::= NK_ID */ yytestcase(yyruleno==213); +{ yylhsminor.yy149 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy149 = yylhsminor.yy149; + break; + case 39: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 42: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + case 40: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 43: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + case 41: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 44: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + case 42: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropQnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 45: /* cmd ::= SHOW QNODES */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL); } + case 43: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy497, &yymsp[-1].minor.yy149, yymsp[0].minor.yy140); } + break; + case 44: /* cmd ::= DROP DATABASE exists_opt db_name */ +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy497, &yymsp[0].minor.yy149); } + break; + case 45: /* cmd ::= USE db_name */ +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy149); } + break; + case 46: /* cmd ::= ALTER DATABASE db_name alter_db_options */ +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy149, yymsp[0].minor.yy140); } + break; + case 47: /* not_exists_opt ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy497 = true; } + break; + case 48: /* not_exists_opt ::= */ + case 50: /* exists_opt ::= */ yytestcase(yyruleno==50); + case 276: /* set_quantifier_opt ::= */ yytestcase(yyruleno==276); +{ yymsp[1].minor.yy497 = false; } + break; + case 49: /* exists_opt ::= IF EXISTS */ +{ yymsp[-1].minor.yy497 = true; } + break; + case 51: /* db_options ::= */ +{ yymsp[1].minor.yy140 = createDefaultDatabaseOptions(pCxt); } + break; + case 52: /* db_options ::= db_options BLOCKS NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_BLOCKS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; + break; + case 53: /* db_options ::= db_options CACHE NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_CACHE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; + break; + case 54: /* db_options ::= db_options CACHELAST NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; + break; + case 55: /* db_options ::= db_options COMP NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; + break; + case 56: /* db_options ::= db_options DAYS NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; + break; + case 57: /* db_options ::= db_options FSYNC NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 46: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy107, &yymsp[-1].minor.yy353, yymsp[0].minor.yy26); } + case 58: /* db_options ::= db_options MAXROWS NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 47: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy107, &yymsp[0].minor.yy353); } + case 59: /* db_options ::= db_options MINROWS NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 48: /* cmd ::= SHOW DATABASES */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } + case 60: /* db_options ::= db_options KEEP NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_KEEP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 49: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy353); } + case 61: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 50: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy353, yymsp[0].minor.yy26); } + case 62: /* db_options ::= db_options QUORUM NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_QUORUM, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 51: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy107 = true; } + case 63: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 52: /* not_exists_opt ::= */ - case 54: /* exists_opt ::= */ yytestcase(yyruleno==54); - case 255: /* set_quantifier_opt ::= */ yytestcase(yyruleno==255); -{ yymsp[1].minor.yy107 = false; } + case 64: /* db_options ::= db_options TTL NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 53: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy107 = true; } + case 65: /* db_options ::= db_options WAL NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 55: /* db_options ::= */ -{ yymsp[1].minor.yy26 = createDefaultDatabaseOptions(pCxt); } + case 66: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 56: /* db_options ::= db_options BLOCKS NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_BLOCKS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 67: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 57: /* db_options ::= db_options CACHE NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_CACHE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 68: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_STREAM_MODE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 58: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 69: /* db_options ::= db_options RETENTIONS NK_STRING */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_RETENTIONS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 59: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 70: /* db_options ::= db_options FILE_FACTOR NK_FLOAT */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-2].minor.yy140, DB_OPTION_FILE_FACTOR, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 60: /* db_options ::= db_options DAYS NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 71: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy140 = createDefaultAlterDatabaseOptions(pCxt); yylhsminor.yy140 = setDatabaseOption(pCxt, yylhsminor.yy140, yymsp[0].minor.yy233.type, &yymsp[0].minor.yy233.val); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 61: /* db_options ::= db_options FSYNC NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 72: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy140 = setDatabaseOption(pCxt, yymsp[-1].minor.yy140, yymsp[0].minor.yy233.type, &yymsp[0].minor.yy233.val); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 62: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 73: /* alter_db_option ::= BLOCKS NK_INTEGER */ +{ yymsp[-1].minor.yy233.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } break; - case 63: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 74: /* alter_db_option ::= FSYNC NK_INTEGER */ +{ yymsp[-1].minor.yy233.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } + break; + case 75: /* alter_db_option ::= KEEP NK_INTEGER */ +{ yymsp[-1].minor.yy233.type = DB_OPTION_KEEP; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } break; - case 64: /* db_options ::= db_options KEEP NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_KEEP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 76: /* alter_db_option ::= WAL NK_INTEGER */ +{ yymsp[-1].minor.yy233.type = DB_OPTION_WAL; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } break; - case 65: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 77: /* alter_db_option ::= QUORUM NK_INTEGER */ +{ yymsp[-1].minor.yy233.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } break; - case 66: /* db_options ::= db_options QUORUM NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_QUORUM, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 78: /* alter_db_option ::= CACHELAST NK_INTEGER */ +{ yymsp[-1].minor.yy233.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } + break; + case 79: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 81: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==81); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy497, yymsp[-5].minor.yy140, yymsp[-3].minor.yy136, yymsp[-1].minor.yy136, yymsp[0].minor.yy140); } + break; + case 80: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy136); } + break; + case 82: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy136); } + break; + case 83: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy497, yymsp[0].minor.yy140); } + break; + case 84: /* cmd ::= ALTER TABLE alter_table_clause */ + case 85: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==85); + case 184: /* cmd ::= query_expression */ yytestcase(yyruleno==184); +{ pCxt->pRootNode = yymsp[0].minor.yy140; } + break; + case 86: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy140 = createAlterTableOption(pCxt, yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; + break; + case 87: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy140 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy140, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy149, yymsp[0].minor.yy256); } + yymsp[-4].minor.yy140 = yylhsminor.yy140; + break; + case 88: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy140 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy140, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy149); } + yymsp[-3].minor.yy140 = yylhsminor.yy140; + break; + case 89: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy140 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy140, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy149, yymsp[0].minor.yy256); } + yymsp[-4].minor.yy140 = yylhsminor.yy140; + break; + case 90: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy140 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy140, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy149, &yymsp[0].minor.yy149); } + yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 67: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 91: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy140 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy140, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy149, yymsp[0].minor.yy256); } + yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 68: /* db_options ::= db_options TTL NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 92: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy140 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy140, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy149); } + yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 69: /* db_options ::= db_options WAL NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 93: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy140 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy140, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy149, yymsp[0].minor.yy256); } + yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 70: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 94: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy140 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy140, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy149, &yymsp[0].minor.yy149); } + yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 71: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 95: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ +{ yylhsminor.yy140 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy140, &yymsp[-2].minor.yy149, yymsp[0].minor.yy140); } + yymsp[-5].minor.yy140 = yylhsminor.yy140; break; - case 72: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_STREAM_MODE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 96: /* multi_create_clause ::= create_subtable_clause */ + case 99: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==99); + case 106: /* column_def_list ::= column_def */ yytestcase(yyruleno==106); + case 147: /* col_name_list ::= col_name */ yytestcase(yyruleno==147); + case 169: /* func_name_list ::= func_name */ yytestcase(yyruleno==169); + case 178: /* func_list ::= func */ yytestcase(yyruleno==178); + case 203: /* literal_list ::= signed_literal */ yytestcase(yyruleno==203); + case 281: /* select_sublist ::= select_item */ yytestcase(yyruleno==281); + case 328: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==328); +{ yylhsminor.yy136 = createNodeList(pCxt, yymsp[0].minor.yy140); } + yymsp[0].minor.yy136 = yylhsminor.yy136; break; - case 73: /* db_options ::= db_options RETENTIONS NK_STRING */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_RETENTIONS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 97: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 100: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==100); +{ yylhsminor.yy136 = addNodeToList(pCxt, yymsp[-1].minor.yy136, yymsp[0].minor.yy140); } + yymsp[-1].minor.yy136 = yylhsminor.yy136; break; - case 74: /* db_options ::= db_options FILE_FACTOR NK_FLOAT */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-2].minor.yy26, DB_OPTION_FILE_FACTOR, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 98: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ +{ yylhsminor.yy140 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy497, yymsp[-7].minor.yy140, yymsp[-5].minor.yy140, yymsp[-4].minor.yy136, yymsp[-1].minor.yy136); } + yymsp[-8].minor.yy140 = yylhsminor.yy140; break; - case 75: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy26 = createDefaultAlterDatabaseOptions(pCxt); yylhsminor.yy26 = setDatabaseOption(pCxt, yylhsminor.yy26, yymsp[0].minor.yy443.type, &yymsp[0].minor.yy443.val); } - yymsp[0].minor.yy26 = yylhsminor.yy26; + case 101: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy140 = createDropTableClause(pCxt, yymsp[-1].minor.yy497, yymsp[0].minor.yy140); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 76: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy26 = setDatabaseOption(pCxt, yymsp[-1].minor.yy26, yymsp[0].minor.yy443.type, &yymsp[0].minor.yy443.val); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + case 102: /* specific_tags_opt ::= */ + case 133: /* tags_def_opt ::= */ yytestcase(yyruleno==133); + case 289: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==289); + case 306: /* group_by_clause_opt ::= */ yytestcase(yyruleno==306); + case 316: /* order_by_clause_opt ::= */ yytestcase(yyruleno==316); +{ yymsp[1].minor.yy136 = NULL; } break; - case 77: /* alter_db_option ::= BLOCKS NK_INTEGER */ -{ yymsp[-1].minor.yy443.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 103: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ +{ yymsp[-2].minor.yy136 = yymsp[-1].minor.yy136; } break; - case 78: /* alter_db_option ::= FSYNC NK_INTEGER */ -{ yymsp[-1].minor.yy443.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 104: /* full_table_name ::= table_name */ +{ yylhsminor.yy140 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy149, NULL); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 79: /* alter_db_option ::= KEEP NK_INTEGER */ -{ yymsp[-1].minor.yy443.type = DB_OPTION_KEEP; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 105: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy140 = createRealTableNode(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149, NULL); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 80: /* alter_db_option ::= WAL NK_INTEGER */ -{ yymsp[-1].minor.yy443.type = DB_OPTION_WAL; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 107: /* column_def_list ::= column_def_list NK_COMMA column_def */ + case 148: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==148); + case 170: /* func_name_list ::= func_name_list NK_COMMA col_name */ yytestcase(yyruleno==170); + case 179: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==179); + case 204: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==204); + case 282: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==282); + case 329: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==329); +{ yylhsminor.yy136 = addNodeToList(pCxt, yymsp[-2].minor.yy136, yymsp[0].minor.yy140); } + yymsp[-2].minor.yy136 = yylhsminor.yy136; break; - case 81: /* alter_db_option ::= QUORUM NK_INTEGER */ -{ yymsp[-1].minor.yy443.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 108: /* column_def ::= column_name type_name */ +{ yylhsminor.yy140 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy149, yymsp[0].minor.yy256, NULL); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 82: /* alter_db_option ::= CACHELAST NK_INTEGER */ -{ yymsp[-1].minor.yy443.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 109: /* column_def ::= column_name type_name COMMENT NK_STRING */ +{ yylhsminor.yy140 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy149, yymsp[-2].minor.yy256, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 83: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 85: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==85); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy107, yymsp[-5].minor.yy26, yymsp[-3].minor.yy64, yymsp[-1].minor.yy64, yymsp[0].minor.yy26); } + case 110: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 84: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy64); } + case 111: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 86: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy64); } + case 112: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 87: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy107, yymsp[0].minor.yy26); } - break; - case 88: /* cmd ::= SHOW TABLES */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } - break; - case 89: /* cmd ::= SHOW STABLES */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } - break; - case 90: /* cmd ::= ALTER TABLE alter_table_clause */ - case 91: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==91); - case 174: /* cmd ::= query_expression */ yytestcase(yyruleno==174); -{ pCxt->pRootNode = yymsp[0].minor.yy26; } - break; - case 92: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy26 = createAlterTableOption(pCxt, yymsp[-1].minor.yy26, yymsp[0].minor.yy26); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; - break; - case 93: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy26 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + case 113: /* type_name ::= INT */ + case 114: /* type_name ::= INTEGER */ yytestcase(yyruleno==114); +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 94: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy26 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy26, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy353); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; + case 115: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 95: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy26 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + case 116: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 96: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy26 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy353, &yymsp[0].minor.yy353); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + case 117: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 97: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy26 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + case 118: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy256 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 98: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy26 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy26, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy353); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; + case 119: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 99: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy26 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + case 120: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy256 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 100: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy26 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy26, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy353, &yymsp[0].minor.yy353); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + case 121: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy256 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 101: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ -{ yylhsminor.yy26 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy26, &yymsp[-2].minor.yy353, yymsp[0].minor.yy26); } - yymsp[-5].minor.yy26 = yylhsminor.yy26; + case 122: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy256 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 102: /* multi_create_clause ::= create_subtable_clause */ - case 105: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==105); - case 112: /* column_def_list ::= column_def */ yytestcase(yyruleno==112); - case 153: /* col_name_list ::= col_name */ yytestcase(yyruleno==153); - case 156: /* func_name_list ::= func_name */ yytestcase(yyruleno==156); - case 165: /* func_list ::= func */ yytestcase(yyruleno==165); - case 260: /* select_sublist ::= select_item */ yytestcase(yyruleno==260); - case 307: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==307); -{ yylhsminor.yy64 = createNodeList(pCxt, yymsp[0].minor.yy26); } - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 123: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy256 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 103: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 106: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==106); -{ yylhsminor.yy64 = addNodeToList(pCxt, yymsp[-1].minor.yy64, yymsp[0].minor.yy26); } - yymsp[-1].minor.yy64 = yylhsminor.yy64; + case 124: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy256 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 104: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ -{ yylhsminor.yy26 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy107, yymsp[-7].minor.yy26, yymsp[-5].minor.yy26, yymsp[-4].minor.yy64, yymsp[-1].minor.yy64); } - yymsp[-8].minor.yy26 = yylhsminor.yy26; + case 125: /* type_name ::= JSON */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 107: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy26 = createDropTableClause(pCxt, yymsp[-1].minor.yy107, yymsp[0].minor.yy26); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + case 126: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy256 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 108: /* specific_tags_opt ::= */ - case 139: /* tags_def_opt ::= */ yytestcase(yyruleno==139); - case 268: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==268); - case 285: /* group_by_clause_opt ::= */ yytestcase(yyruleno==285); - case 295: /* order_by_clause_opt ::= */ yytestcase(yyruleno==295); -{ yymsp[1].minor.yy64 = NULL; } + case 127: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 109: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; } + case 128: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 110: /* full_table_name ::= table_name */ -{ yylhsminor.yy26 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy353, NULL); } - yymsp[0].minor.yy26 = yylhsminor.yy26; + case 129: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy256 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 111: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy26 = createRealTableNode(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy353, NULL); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 130: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy256 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 113: /* column_def_list ::= column_def_list NK_COMMA column_def */ - case 154: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==154); - case 157: /* func_name_list ::= func_name_list NK_COMMA col_name */ yytestcase(yyruleno==157); - case 166: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==166); - case 261: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==261); - case 308: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==308); -{ yylhsminor.yy64 = addNodeToList(pCxt, yymsp[-2].minor.yy64, yymsp[0].minor.yy26); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 131: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy256 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 114: /* column_def ::= column_name type_name */ -{ yylhsminor.yy26 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy353, yymsp[0].minor.yy370, NULL); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + case 132: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy256 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 115: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy26 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy353, yymsp[-2].minor.yy370, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; + case 134: /* tags_def_opt ::= tags_def */ + case 280: /* select_list ::= select_sublist */ yytestcase(yyruleno==280); +{ yylhsminor.yy136 = yymsp[0].minor.yy136; } + yymsp[0].minor.yy136 = yylhsminor.yy136; break; - case 116: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 135: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ +{ yymsp[-3].minor.yy136 = yymsp[-1].minor.yy136; } break; - case 117: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 136: /* table_options ::= */ +{ yymsp[1].minor.yy140 = createDefaultTableOptions(pCxt); } break; - case 118: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 137: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy140 = setTableOption(pCxt, yymsp[-2].minor.yy140, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 119: /* type_name ::= INT */ - case 120: /* type_name ::= INTEGER */ yytestcase(yyruleno==120); -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_INT); } + case 138: /* table_options ::= table_options KEEP NK_INTEGER */ +{ yylhsminor.yy140 = setTableOption(pCxt, yymsp[-2].minor.yy140, TABLE_OPTION_KEEP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 121: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 139: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy140 = setTableOption(pCxt, yymsp[-2].minor.yy140, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 122: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 140: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy140 = setTableSmaOption(pCxt, yymsp[-4].minor.yy140, yymsp[-1].minor.yy136); } + yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 123: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 141: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ +{ yylhsminor.yy140 = setTableRollupOption(pCxt, yymsp[-4].minor.yy140, yymsp[-1].minor.yy136); } + yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 124: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy370 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 142: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy140 = createDefaultAlterTableOptions(pCxt); yylhsminor.yy140 = setTableOption(pCxt, yylhsminor.yy140, yymsp[0].minor.yy233.type, &yymsp[0].minor.yy233.val); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 125: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 143: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy140 = setTableOption(pCxt, yymsp[-1].minor.yy140, yymsp[0].minor.yy233.type, &yymsp[0].minor.yy233.val); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 126: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy370 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 144: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy233.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } break; - case 127: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy370 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 145: /* alter_table_option ::= KEEP NK_INTEGER */ +{ yymsp[-1].minor.yy233.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } break; - case 128: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy370 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 146: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy233.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } break; - case 129: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy370 = createDataType(TSDB_DATA_TYPE_UINT); } + case 149: /* col_name ::= column_name */ +{ yylhsminor.yy140 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy149); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 130: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy370 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 150: /* cmd ::= SHOW DNODES */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL, NULL); } break; - case 131: /* type_name ::= JSON */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_JSON); } + case 151: /* cmd ::= SHOW USERS */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL, NULL); } break; - case 132: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy370 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 152: /* cmd ::= SHOW DATABASES */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL, NULL); } break; - case 133: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 153: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy140, yymsp[0].minor.yy140); } break; - case 134: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 154: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy140, yymsp[0].minor.yy140); } break; - case 135: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy370 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 155: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy140, NULL); } break; - case 136: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy370 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 156: /* cmd ::= SHOW MNODES */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL, NULL); } break; - case 137: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy370 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 157: /* cmd ::= SHOW MODULES */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MODULES_STMT, NULL, NULL); } break; - case 138: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy370 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 158: /* cmd ::= SHOW QNODES */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL, NULL); } break; - case 140: /* tags_def_opt ::= tags_def */ - case 259: /* select_list ::= select_sublist */ yytestcase(yyruleno==259); -{ yylhsminor.yy64 = yymsp[0].minor.yy64; } - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 159: /* cmd ::= SHOW FUNCTIONS */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } break; - case 141: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy64 = yymsp[-1].minor.yy64; } + case 160: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 142: /* table_options ::= */ -{ yymsp[1].minor.yy26 = createDefaultTableOptions(pCxt); } + case 161: /* cmd ::= SHOW STREAMS */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } break; - case 143: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy26 = setTableOption(pCxt, yymsp[-2].minor.yy26, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 162: /* db_name_cond_opt ::= */ + case 167: /* from_db_opt ::= */ yytestcase(yyruleno==167); +{ yymsp[1].minor.yy140 = createDefaultDatabaseCondValue(pCxt); } break; - case 144: /* table_options ::= table_options KEEP NK_INTEGER */ -{ yylhsminor.yy26 = setTableOption(pCxt, yymsp[-2].minor.yy26, TABLE_OPTION_KEEP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 163: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy149); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 145: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy26 = setTableOption(pCxt, yymsp[-2].minor.yy26, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 164: /* like_pattern_opt ::= */ + case 175: /* index_options ::= */ yytestcase(yyruleno==175); + case 287: /* where_clause_opt ::= */ yytestcase(yyruleno==287); + case 291: /* twindow_clause_opt ::= */ yytestcase(yyruleno==291); + case 296: /* sliding_opt ::= */ yytestcase(yyruleno==296); + case 298: /* fill_opt ::= */ yytestcase(yyruleno==298); + case 310: /* having_clause_opt ::= */ yytestcase(yyruleno==310); + case 318: /* slimit_clause_opt ::= */ yytestcase(yyruleno==318); + case 322: /* limit_clause_opt ::= */ yytestcase(yyruleno==322); +{ yymsp[1].minor.yy140 = NULL; } break; - case 146: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy26 = setTableSmaOption(pCxt, yymsp[-4].minor.yy26, yymsp[-1].minor.yy64); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + case 165: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 147: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ -{ yylhsminor.yy26 = setTableRollupOption(pCxt, yymsp[-4].minor.yy26, yymsp[-1].minor.yy64); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + case 166: /* table_name_cond ::= table_name */ +{ yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy149); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 148: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy26 = createDefaultAlterTableOptions(pCxt); yylhsminor.yy26 = setTableOption(pCxt, yylhsminor.yy26, yymsp[0].minor.yy443.type, &yymsp[0].minor.yy443.val); } - yymsp[0].minor.yy26 = yylhsminor.yy26; + case 168: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy149); } break; - case 149: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy26 = setTableOption(pCxt, yymsp[-1].minor.yy26, yymsp[0].minor.yy443.type, &yymsp[0].minor.yy443.val); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + case 171: /* func_name ::= function_name */ +{ yylhsminor.yy140 = createFunctionNode(pCxt, &yymsp[0].minor.yy149, NULL); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 150: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy443.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 172: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy497, &yymsp[-3].minor.yy149, &yymsp[-1].minor.yy149, NULL, yymsp[0].minor.yy140); } break; - case 151: /* alter_table_option ::= KEEP NK_INTEGER */ -{ yymsp[-1].minor.yy443.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 173: /* cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy497, &yymsp[-5].minor.yy149, &yymsp[-3].minor.yy149, yymsp[-1].minor.yy136, NULL); } break; - case 152: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy443.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy443.val = yymsp[0].minor.yy0; } + case 174: /* cmd ::= DROP INDEX exists_opt index_name ON table_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy497, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149); } break; - case 155: /* col_name ::= column_name */ -{ yylhsminor.yy26 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy353); } - yymsp[0].minor.yy26 = yylhsminor.yy26; + case 176: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ +{ yymsp[-8].minor.yy140 = createIndexOption(pCxt, yymsp[-6].minor.yy136, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), NULL, yymsp[0].minor.yy140); } break; - case 158: /* func_name ::= function_name */ -{ yylhsminor.yy26 = createFunctionNode(pCxt, &yymsp[0].minor.yy353, NULL); } - yymsp[0].minor.yy26 = yylhsminor.yy26; + case 177: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ +{ yymsp[-10].minor.yy140 = createIndexOption(pCxt, yymsp[-8].minor.yy136, releaseRawExprNode(pCxt, yymsp[-4].minor.yy140), releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), yymsp[0].minor.yy140); } break; - case 159: /* cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &yymsp[-3].minor.yy353, &yymsp[-1].minor.yy353, NULL, yymsp[0].minor.yy26); } + case 180: /* func ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy140 = createFunctionNode(pCxt, &yymsp[-3].minor.yy149, yymsp[-1].minor.yy136); } + yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 160: /* cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &yymsp[-5].minor.yy353, &yymsp[-3].minor.yy353, yymsp[-1].minor.yy64, NULL); } + case 181: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy497, &yymsp[-2].minor.yy149, yymsp[0].minor.yy140, NULL); } break; - case 161: /* cmd ::= DROP INDEX index_name ON table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy353); } + case 182: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy497, &yymsp[-2].minor.yy149, NULL, &yymsp[0].minor.yy149); } break; - case 162: /* index_options ::= */ - case 266: /* where_clause_opt ::= */ yytestcase(yyruleno==266); - case 270: /* twindow_clause_opt ::= */ yytestcase(yyruleno==270); - case 275: /* sliding_opt ::= */ yytestcase(yyruleno==275); - case 277: /* fill_opt ::= */ yytestcase(yyruleno==277); - case 289: /* having_clause_opt ::= */ yytestcase(yyruleno==289); - case 297: /* slimit_clause_opt ::= */ yytestcase(yyruleno==297); - case 301: /* limit_clause_opt ::= */ yytestcase(yyruleno==301); -{ yymsp[1].minor.yy26 = NULL; } + case 183: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy497, &yymsp[0].minor.yy149); } break; - case 163: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ -{ yymsp[-8].minor.yy26 = createIndexOption(pCxt, yymsp[-6].minor.yy64, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), NULL, yymsp[0].minor.yy26); } + case 185: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 164: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ -{ yymsp[-10].minor.yy26 = createIndexOption(pCxt, yymsp[-8].minor.yy64, releaseRawExprNode(pCxt, yymsp[-4].minor.yy26), releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), yymsp[0].minor.yy26); } + case 186: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 167: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy26 = createFunctionNode(pCxt, &yymsp[-3].minor.yy353, yymsp[-1].minor.yy64); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; + case 187: /* literal ::= NK_STRING */ +{ yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 168: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy107, &yymsp[-2].minor.yy353, yymsp[0].minor.yy26, NULL); } + case 188: /* literal ::= NK_BOOL */ +{ yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 169: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy107, &yymsp[-2].minor.yy353, NULL, &yymsp[0].minor.yy353); } + case 189: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 170: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy107, &yymsp[0].minor.yy353); } + case 190: /* literal ::= duration_literal */ + case 198: /* signed_literal ::= signed */ yytestcase(yyruleno==198); + case 214: /* expression ::= literal */ yytestcase(yyruleno==214); + case 215: /* expression ::= column_reference */ yytestcase(yyruleno==215); + case 218: /* expression ::= subquery */ yytestcase(yyruleno==218); + case 250: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==250); + case 254: /* boolean_primary ::= predicate */ yytestcase(yyruleno==254); + case 256: /* common_expression ::= expression */ yytestcase(yyruleno==256); + case 257: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==257); + case 259: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==259); + case 261: /* table_reference ::= table_primary */ yytestcase(yyruleno==261); + case 262: /* table_reference ::= joined_table */ yytestcase(yyruleno==262); + case 266: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==266); + case 313: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==313); + case 315: /* query_primary ::= query_specification */ yytestcase(yyruleno==315); +{ yylhsminor.yy140 = yymsp[0].minor.yy140; } + yymsp[0].minor.yy140 = yylhsminor.yy140; + break; + case 191: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy140 = yylhsminor.yy140; + break; + case 192: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy140 = yylhsminor.yy140; + break; + case 193: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 194: /* signed ::= NK_MINUS NK_INTEGER */ +{ + SToken t = yymsp[-1].minor.yy0; + t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; + yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + } + yymsp[-1].minor.yy140 = yylhsminor.yy140; + break; + case 195: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy140 = yylhsminor.yy140; + break; + case 196: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + break; + case 197: /* signed ::= NK_MINUS NK_FLOAT */ +{ + SToken t = yymsp[-1].minor.yy0; + t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; + yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + } + yymsp[-1].minor.yy140 = yylhsminor.yy140; + break; + case 199: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy140 = yylhsminor.yy140; + break; + case 200: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 171: /* cmd ::= SHOW VGROUPS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, NULL); } - break; - case 172: /* cmd ::= SHOW db_name NK_DOT VGROUPS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, &yymsp[-2].minor.yy353); } - break; - case 173: /* cmd ::= SHOW MNODES */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL); } - break; - case 175: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy26 = yylhsminor.yy26; - break; - case 176: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy26 = yylhsminor.yy26; - break; - case 177: /* literal ::= NK_STRING */ -{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy26 = yylhsminor.yy26; - break; - case 178: /* literal ::= NK_BOOL */ -{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy26 = yylhsminor.yy26; - break; - case 179: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; - break; - case 180: /* literal ::= duration_literal */ - case 193: /* expression ::= literal */ yytestcase(yyruleno==193); - case 194: /* expression ::= column_reference */ yytestcase(yyruleno==194); - case 197: /* expression ::= subquery */ yytestcase(yyruleno==197); - case 229: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==229); - case 233: /* boolean_primary ::= predicate */ yytestcase(yyruleno==233); - case 235: /* common_expression ::= expression */ yytestcase(yyruleno==235); - case 236: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==236); - case 238: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==238); - case 240: /* table_reference ::= table_primary */ yytestcase(yyruleno==240); - case 241: /* table_reference ::= joined_table */ yytestcase(yyruleno==241); - case 245: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==245); - case 292: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==292); - case 294: /* query_primary ::= query_specification */ yytestcase(yyruleno==294); -{ yylhsminor.yy26 = yymsp[0].minor.yy26; } - yymsp[0].minor.yy26 = yylhsminor.yy26; - break; - case 181: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy26 = yylhsminor.yy26; - break; - case 182: /* literal_list ::= literal */ - case 206: /* expression_list ::= expression */ yytestcase(yyruleno==206); -{ yylhsminor.yy64 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy26)); } - yymsp[0].minor.yy64 = yylhsminor.yy64; - break; - case 183: /* literal_list ::= literal_list NK_COMMA literal */ - case 207: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==207); -{ yylhsminor.yy64 = addNodeToList(pCxt, yymsp[-2].minor.yy64, releaseRawExprNode(pCxt, yymsp[0].minor.yy26)); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; - break; - case 195: /* expression ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy353, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy353, yymsp[-1].minor.yy64)); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; - break; - case 196: /* expression ::= function_name NK_LP NK_STAR NK_RP */ -{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy353, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy353, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; - break; - case 198: /* expression ::= NK_LP expression NK_RP */ - case 234: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==234); -{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy26)); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; - break; - case 199: /* expression ::= NK_PLUS expression */ + case 201: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 202: /* signed_literal ::= duration_literal */ + case 327: /* search_condition ::= common_expression */ yytestcase(yyruleno==327); +{ yylhsminor.yy140 = releaseRawExprNode(pCxt, yymsp[0].minor.yy140); } + yymsp[0].minor.yy140 = yylhsminor.yy140; + break; + case 216: /* expression ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy149, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy149, yymsp[-1].minor.yy136)); } + yymsp[-3].minor.yy140 = yylhsminor.yy140; + break; + case 217: /* expression ::= function_name NK_LP NK_STAR NK_RP */ +{ yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy149, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy149, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } + yymsp[-3].minor.yy140 = yylhsminor.yy140; + break; + case 219: /* expression ::= NK_LP expression NK_RP */ + case 255: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==255); +{ yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; + break; + case 220: /* expression ::= NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy26)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 200: /* expression ::= NK_MINUS expression */ + case 221: /* expression ::= NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy26), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy140), NULL)); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 201: /* expression ::= expression NK_PLUS expression */ + case 222: /* expression ::= expression NK_PLUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 202: /* expression ::= expression NK_MINUS expression */ + case 223: /* expression ::= expression NK_MINUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 203: /* expression ::= expression NK_STAR expression */ + case 224: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 204: /* expression ::= expression NK_SLASH expression */ + case 225: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 205: /* expression ::= expression NK_REM expression */ + case 226: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; + break; + case 227: /* expression_list ::= expression */ +{ yylhsminor.yy136 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } + yymsp[0].minor.yy136 = yylhsminor.yy136; break; - case 208: /* column_reference ::= column_name */ -{ yylhsminor.yy26 = createRawExprNode(pCxt, &yymsp[0].minor.yy353, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy353)); } - yymsp[0].minor.yy26 = yylhsminor.yy26; + case 228: /* expression_list ::= expression_list NK_COMMA expression */ +{ yylhsminor.yy136 = addNodeToList(pCxt, yymsp[-2].minor.yy136, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } + yymsp[-2].minor.yy136 = yylhsminor.yy136; break; - case 209: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy353, createColumnNode(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy353)); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 229: /* column_reference ::= column_name */ +{ yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy149, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy149)); } + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 210: /* predicate ::= expression compare_op expression */ - case 215: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==215); + case 230: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149, createColumnNode(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149)); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; + break; + case 231: /* predicate ::= expression compare_op expression */ + case 236: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==236); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy80, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy320, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 211: /* predicate ::= expression BETWEEN expression AND expression */ + case 232: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy26), releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy140), releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-4].minor.yy26 = yylhsminor.yy26; + yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 212: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 233: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[-5].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[-5].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-5].minor.yy26 = yylhsminor.yy26; + yymsp[-5].minor.yy140 = yylhsminor.yy140; break; - case 213: /* predicate ::= expression IS NULL */ + case 234: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), NULL)); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 214: /* predicate ::= expression IS NOT NULL */ + case 235: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy26), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), NULL)); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; + yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 216: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy80 = OP_TYPE_LOWER_THAN; } + case 237: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy320 = OP_TYPE_LOWER_THAN; } break; - case 217: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy80 = OP_TYPE_GREATER_THAN; } + case 238: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy320 = OP_TYPE_GREATER_THAN; } break; - case 218: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy80 = OP_TYPE_LOWER_EQUAL; } + case 239: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy320 = OP_TYPE_LOWER_EQUAL; } break; - case 219: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy80 = OP_TYPE_GREATER_EQUAL; } + case 240: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy320 = OP_TYPE_GREATER_EQUAL; } break; - case 220: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy80 = OP_TYPE_NOT_EQUAL; } + case 241: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy320 = OP_TYPE_NOT_EQUAL; } break; - case 221: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy80 = OP_TYPE_EQUAL; } + case 242: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy320 = OP_TYPE_EQUAL; } break; - case 222: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy80 = OP_TYPE_LIKE; } + case 243: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy320 = OP_TYPE_LIKE; } break; - case 223: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy80 = OP_TYPE_NOT_LIKE; } + case 244: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy320 = OP_TYPE_NOT_LIKE; } break; - case 224: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy80 = OP_TYPE_MATCH; } + case 245: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy320 = OP_TYPE_MATCH; } break; - case 225: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy80 = OP_TYPE_NMATCH; } + case 246: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy320 = OP_TYPE_NMATCH; } break; - case 226: /* in_op ::= IN */ -{ yymsp[0].minor.yy80 = OP_TYPE_IN; } + case 247: /* in_op ::= IN */ +{ yymsp[0].minor.yy320 = OP_TYPE_IN; } break; - case 227: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy80 = OP_TYPE_NOT_IN; } + case 248: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy320 = OP_TYPE_NOT_IN; } break; - case 228: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy64)); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 249: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy136)); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 230: /* boolean_value_expression ::= NOT boolean_primary */ + case 251: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy26), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy140), NULL)); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 231: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 252: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 232: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 253: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy26); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 237: /* from_clause ::= FROM table_reference_list */ - case 267: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==267); - case 290: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==290); -{ yymsp[-1].minor.yy26 = yymsp[0].minor.yy26; } + case 258: /* from_clause ::= FROM table_reference_list */ + case 288: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==288); + case 311: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==311); +{ yymsp[-1].minor.yy140 = yymsp[0].minor.yy140; } break; - case 239: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy26 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy26, yymsp[0].minor.yy26, NULL); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 260: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy140 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy140, yymsp[0].minor.yy140, NULL); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 242: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy26 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy353, &yymsp[0].minor.yy353); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + case 263: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy140 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy149, &yymsp[0].minor.yy149); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 243: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy26 = createRealTableNode(pCxt, &yymsp[-3].minor.yy353, &yymsp[-1].minor.yy353, &yymsp[0].minor.yy353); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; + case 264: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy140 = createRealTableNode(pCxt, &yymsp[-3].minor.yy149, &yymsp[-1].minor.yy149, &yymsp[0].minor.yy149); } + yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 244: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy26 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy26), &yymsp[0].minor.yy353); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + case 265: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy140 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140), &yymsp[0].minor.yy149); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 246: /* alias_opt ::= */ -{ yymsp[1].minor.yy353 = nil_token; } + case 267: /* alias_opt ::= */ +{ yymsp[1].minor.yy149 = nil_token; } break; - case 247: /* alias_opt ::= table_alias */ -{ yylhsminor.yy353 = yymsp[0].minor.yy353; } - yymsp[0].minor.yy353 = yylhsminor.yy353; + case 268: /* alias_opt ::= table_alias */ +{ yylhsminor.yy149 = yymsp[0].minor.yy149; } + yymsp[0].minor.yy149 = yylhsminor.yy149; break; - case 248: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy353 = yymsp[0].minor.yy353; } + case 269: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy149 = yymsp[0].minor.yy149; } break; - case 249: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 250: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==250); -{ yymsp[-2].minor.yy26 = yymsp[-1].minor.yy26; } + case 270: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 271: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==271); +{ yymsp[-2].minor.yy140 = yymsp[-1].minor.yy140; } break; - case 251: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy26 = createJoinTableNode(pCxt, yymsp[-4].minor.yy372, yymsp[-5].minor.yy26, yymsp[-2].minor.yy26, yymsp[0].minor.yy26); } - yymsp[-5].minor.yy26 = yylhsminor.yy26; + case 272: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy140 = createJoinTableNode(pCxt, yymsp[-4].minor.yy144, yymsp[-5].minor.yy140, yymsp[-2].minor.yy140, yymsp[0].minor.yy140); } + yymsp[-5].minor.yy140 = yylhsminor.yy140; break; - case 252: /* join_type ::= */ -{ yymsp[1].minor.yy372 = JOIN_TYPE_INNER; } + case 273: /* join_type ::= */ +{ yymsp[1].minor.yy144 = JOIN_TYPE_INNER; } break; - case 253: /* join_type ::= INNER */ -{ yymsp[0].minor.yy372 = JOIN_TYPE_INNER; } + case 274: /* join_type ::= INNER */ +{ yymsp[0].minor.yy144 = JOIN_TYPE_INNER; } break; - case 254: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 275: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-8].minor.yy26 = createSelectStmt(pCxt, yymsp[-7].minor.yy107, yymsp[-6].minor.yy64, yymsp[-5].minor.yy26); - yymsp[-8].minor.yy26 = addWhereClause(pCxt, yymsp[-8].minor.yy26, yymsp[-4].minor.yy26); - yymsp[-8].minor.yy26 = addPartitionByClause(pCxt, yymsp[-8].minor.yy26, yymsp[-3].minor.yy64); - yymsp[-8].minor.yy26 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy26, yymsp[-2].minor.yy26); - yymsp[-8].minor.yy26 = addGroupByClause(pCxt, yymsp[-8].minor.yy26, yymsp[-1].minor.yy64); - yymsp[-8].minor.yy26 = addHavingClause(pCxt, yymsp[-8].minor.yy26, yymsp[0].minor.yy26); + yymsp[-8].minor.yy140 = createSelectStmt(pCxt, yymsp[-7].minor.yy497, yymsp[-6].minor.yy136, yymsp[-5].minor.yy140); + yymsp[-8].minor.yy140 = addWhereClause(pCxt, yymsp[-8].minor.yy140, yymsp[-4].minor.yy140); + yymsp[-8].minor.yy140 = addPartitionByClause(pCxt, yymsp[-8].minor.yy140, yymsp[-3].minor.yy136); + yymsp[-8].minor.yy140 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy140, yymsp[-2].minor.yy140); + yymsp[-8].minor.yy140 = addGroupByClause(pCxt, yymsp[-8].minor.yy140, yymsp[-1].minor.yy136); + yymsp[-8].minor.yy140 = addHavingClause(pCxt, yymsp[-8].minor.yy140, yymsp[0].minor.yy140); } break; - case 256: /* set_quantifier_opt ::= DISTINCT */ -{ yymsp[0].minor.yy107 = true; } + case 277: /* set_quantifier_opt ::= DISTINCT */ +{ yymsp[0].minor.yy497 = true; } break; - case 257: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy107 = false; } + case 278: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy497 = false; } break; - case 258: /* select_list ::= NK_STAR */ -{ yymsp[0].minor.yy64 = NULL; } + case 279: /* select_list ::= NK_STAR */ +{ yymsp[0].minor.yy136 = NULL; } break; - case 262: /* select_item ::= common_expression */ + case 283: /* select_item ::= common_expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy26); - yylhsminor.yy26 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy26), &t); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); + yylhsminor.yy140 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy140), &t); } - yymsp[0].minor.yy26 = yylhsminor.yy26; + yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 263: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy26 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy26), &yymsp[0].minor.yy353); } - yymsp[-1].minor.yy26 = yylhsminor.yy26; + case 284: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy140 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140), &yymsp[0].minor.yy149); } + yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 264: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy26 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), &yymsp[0].minor.yy353); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 285: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy140 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), &yymsp[0].minor.yy149); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 265: /* select_item ::= table_name NK_DOT NK_STAR */ -{ yylhsminor.yy26 = createColumnNode(pCxt, &yymsp[-2].minor.yy353, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 286: /* select_item ::= table_name NK_DOT NK_STAR */ +{ yylhsminor.yy140 = createColumnNode(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 269: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 286: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==286); - case 296: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==296); -{ yymsp[-2].minor.yy64 = yymsp[0].minor.yy64; } + case 290: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 307: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==307); + case 317: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==317); +{ yymsp[-2].minor.yy136 = yymsp[0].minor.yy136; } break; - case 271: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy26 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy26), &yymsp[-1].minor.yy0); } + case 292: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy140 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } break; - case 272: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ -{ yymsp[-3].minor.yy26 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy26)); } + case 293: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ yymsp[-3].minor.yy140 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } break; - case 273: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy26 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy26), NULL, yymsp[-1].minor.yy26, yymsp[0].minor.yy26); } + case 294: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy140 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), NULL, yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 274: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy26 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy26), releaseRawExprNode(pCxt, yymsp[-3].minor.yy26), yymsp[-1].minor.yy26, yymsp[0].minor.yy26); } + case 295: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy140 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy140), releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 276: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ yymsp[-3].minor.yy26 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy26); } + case 297: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ yymsp[-3].minor.yy140 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy140); } break; - case 278: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy26 = createFillNode(pCxt, yymsp[-1].minor.yy192, NULL); } + case 299: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy140 = createFillNode(pCxt, yymsp[-1].minor.yy306, NULL); } break; - case 279: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy26 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy64)); } + case 300: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy140 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy136)); } break; - case 280: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy192 = FILL_MODE_NONE; } + case 301: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy306 = FILL_MODE_NONE; } break; - case 281: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy192 = FILL_MODE_PREV; } + case 302: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy306 = FILL_MODE_PREV; } break; - case 282: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy192 = FILL_MODE_NULL; } + case 303: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy306 = FILL_MODE_NULL; } break; - case 283: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy192 = FILL_MODE_LINEAR; } + case 304: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy306 = FILL_MODE_LINEAR; } break; - case 284: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy192 = FILL_MODE_NEXT; } + case 305: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy306 = FILL_MODE_NEXT; } break; - case 287: /* group_by_list ::= expression */ -{ yylhsminor.yy64 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 308: /* group_by_list ::= expression */ +{ yylhsminor.yy136 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } + yymsp[0].minor.yy136 = yylhsminor.yy136; break; - case 288: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy64 = addNodeToList(pCxt, yymsp[-2].minor.yy64, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy26))); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 309: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy136 = addNodeToList(pCxt, yymsp[-2].minor.yy136, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } + yymsp[-2].minor.yy136 = yylhsminor.yy136; break; - case 291: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 312: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy26 = addOrderByClause(pCxt, yymsp[-3].minor.yy26, yymsp[-2].minor.yy64); - yylhsminor.yy26 = addSlimitClause(pCxt, yylhsminor.yy26, yymsp[-1].minor.yy26); - yylhsminor.yy26 = addLimitClause(pCxt, yylhsminor.yy26, yymsp[0].minor.yy26); + yylhsminor.yy140 = addOrderByClause(pCxt, yymsp[-3].minor.yy140, yymsp[-2].minor.yy136); + yylhsminor.yy140 = addSlimitClause(pCxt, yylhsminor.yy140, yymsp[-1].minor.yy140); + yylhsminor.yy140 = addLimitClause(pCxt, yylhsminor.yy140, yymsp[0].minor.yy140); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; - break; - case 293: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy26 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy26, yymsp[0].minor.yy26); } - yymsp[-3].minor.yy26 = yylhsminor.yy26; + yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 298: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 302: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==302); -{ yymsp[-1].minor.yy26 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 314: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy140 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy140, yymsp[0].minor.yy140); } + yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 299: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 303: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==303); -{ yymsp[-3].minor.yy26 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 319: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 323: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==323); +{ yymsp[-1].minor.yy140 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 300: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 304: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==304); -{ yymsp[-3].minor.yy26 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 320: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 324: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==324); +{ yymsp[-3].minor.yy140 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 305: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy26 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy26); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 321: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 325: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==325); +{ yymsp[-3].minor.yy140 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 306: /* search_condition ::= common_expression */ -{ yylhsminor.yy26 = releaseRawExprNode(pCxt, yymsp[0].minor.yy26); } - yymsp[0].minor.yy26 = yylhsminor.yy26; + case 326: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy140); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 309: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy26 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy26), yymsp[-1].minor.yy32, yymsp[0].minor.yy391); } - yymsp[-2].minor.yy26 = yylhsminor.yy26; + case 330: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy140 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), yymsp[-1].minor.yy158, yymsp[0].minor.yy73); } + yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 310: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy32 = ORDER_ASC; } + case 331: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy158 = ORDER_ASC; } break; - case 311: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy32 = ORDER_ASC; } + case 332: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy158 = ORDER_ASC; } break; - case 312: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy32 = ORDER_DESC; } + case 333: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy158 = ORDER_DESC; } break; - case 313: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy391 = NULL_ORDER_DEFAULT; } + case 334: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy73 = NULL_ORDER_DEFAULT; } break; - case 314: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy391 = NULL_ORDER_FIRST; } + case 335: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy73 = NULL_ORDER_FIRST; } break; - case 315: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy391 = NULL_ORDER_LAST; } + case 336: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy73 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 3236ad165c3aef025505e8802a26efab27ed63e7..5723b93f8b50e89f9228697e0b9101db02a749b7 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -26,6 +26,64 @@ #include "mockCatalog.h" namespace { +void generateInformationSchema(MockCatalogService* mcs) { + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "dnodes", TSDB_SYSTEM_TABLE, 1).addColumn("id", TSDB_DATA_TYPE_INT); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "mnodes", TSDB_SYSTEM_TABLE, 1).addColumn("id", TSDB_DATA_TYPE_INT); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "modules", TSDB_SYSTEM_TABLE, 1).addColumn("id", TSDB_DATA_TYPE_INT); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "qnodes", TSDB_SYSTEM_TABLE, 1).addColumn("id", TSDB_DATA_TYPE_INT); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "user_databases", TSDB_SYSTEM_TABLE, 1).addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "user_functions", TSDB_SYSTEM_TABLE, 1).addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_FUNC_NAME_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "user_indexes", TSDB_SYSTEM_TABLE, 2) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN).addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "user_stables", TSDB_SYSTEM_TABLE, 2) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN).addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "user_streams", TSDB_SYSTEM_TABLE, 1).addColumn("stream_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "user_tables", TSDB_SYSTEM_TABLE, 2) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN).addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "user_table_distributed", TSDB_SYSTEM_TABLE, 1).addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "user_users", TSDB_SYSTEM_TABLE, 1).addColumn("user_name", TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "vgroups", TSDB_SYSTEM_TABLE, 1).addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); + builder.done(); + } +} + void generateTestT1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) @@ -66,6 +124,10 @@ int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* ve return 0; } +int32_t __catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { + return 0; +} + void initMetaDataEnv() { mockCatalogService.reset(new MockCatalogService()); @@ -74,6 +136,8 @@ void initMetaDataEnv() { stub.set(catalogGetTableMeta, __catalogGetTableMeta); stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); stub.set(catalogGetTableDistVgInfo, __catalogGetTableDistVgInfo); + stub.set(catalogGetDBVgVersion, __catalogGetDBVgVersion); + stub.set(catalogGetDBVgInfo, __catalogGetDBVgInfo); // { // AddrAny any("libcatalog.so"); // std::map result; @@ -117,6 +181,7 @@ void initMetaDataEnv() { } void generateMetaData() { + generateInformationSchema(mockCatalogService.get()); generateTestT1(mockCatalogService.get()); generateTestST1(mockCatalogService.get()); mockCatalogService->showTables(); diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 1f333e49e7726f90567d437731c27b878e449046..012af26c17ce190c79c969960d6206a12de28f3b 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -16,8 +16,8 @@ #include #include #include -#include "tdatablock.h" #include "mockCatalogService.h" +#include "tdatablock.h" #include "tname.h" #include "ttypes.h" @@ -65,7 +65,7 @@ private: friend class MockCatalogServiceImpl; static std::unique_ptr createTableBuilder(int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { - STableMeta* meta = (STableMeta*)std::calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags)); + STableMeta* meta = (STableMeta*)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags)); if (nullptr == meta) { throw std::bad_alloc(); } @@ -120,23 +120,15 @@ public: } int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { - // todo - vgInfo->vgId = 1; - addEpIntoEpSet(&vgInfo->epSet, "node1", 6030); - return 0; + char db[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(pTableName, db); + return copyTableVgroup(db, tNameGetTableName(pTableName), vgInfo); } - int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const { - SVgroupInfo info = {0}; - info.vgId = 1; - addEpIntoEpSet(&info.epSet, "node1", 6030); - - info.hashBegin = 0; - info.hashEnd = 1; - *pVgList = taosArrayInit(4, sizeof(SVgroupInfo)); - - taosArrayPush(*pVgList, &info); - return 0; + int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** vgList) const { + char db[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(pTableName, db); + return copyTableVgroup(db, tNameGetTableName(pTableName), vgList); } TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { @@ -285,7 +277,7 @@ private: return TSDB_CODE_TSC_INVALID_TABLE_NAME; } int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); - dst->reset((STableMeta*)std::calloc(1, len)); + dst->reset((STableMeta*)taosMemoryCalloc(1, len)); if (!dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -293,6 +285,27 @@ private: return TSDB_CODE_SUCCESS; } + int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SVgroupInfo* vg) const { + std::shared_ptr table = getTableMeta(db, tbname); + if (table->vgs.empty()) { + return TSDB_CODE_SUCCESS; + } + memcpy(vg, &(table->vgs[0]), sizeof(SVgroupInfo)); + return TSDB_CODE_SUCCESS; + } + + int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SArray** vgList) const { + std::shared_ptr table = getTableMeta(db, tbname); + if (table->vgs.empty()) { + return TSDB_CODE_SUCCESS; + } + *vgList = taosArrayInit(table->vgs.size(), sizeof(SVgroupInfo)); + for (const SVgroupInfo& vg : table->vgs) { + taosArrayPush(*vgList, &vg); + } + return TSDB_CODE_SUCCESS; + } + uint64_t id_; std::unique_ptr builder_; DbMetaCache meta_; diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index a722f0b821255d9564f298842aa6d92afd37f0ae..887979f11e47af79a58ba7d94be4a6e46fbc17c2 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -44,7 +44,7 @@ public: struct MockTableMeta { ~MockTableMeta() { - free(schema); + taosMemoryFree(schema); } STableMeta* schema; diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index bb2f2539e90c5d2887228cb27d59083c3085525e..1813ad0827fc47207224de624dfaaf1468cfecc8 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -44,6 +44,9 @@ protected: query_ = nullptr; bool res = runImpl(parseCode, translateCode); qDestroyQuery(query_); + if (!res) { + dump(); + } return res; } @@ -53,26 +56,40 @@ private: bool runImpl(int32_t parseCode, int32_t translateCode) { int32_t code = doParse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] parser code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - return (TSDB_CODE_SUCCESS != parseCode); + parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; + return (terrno == parseCode); } if (TSDB_CODE_SUCCESS != parseCode) { return false; } - string parserStr = toString(query_->pRoot); + parsedAstStr_ = toString(query_->pRoot); code = doTranslate(&cxt_, query_); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] translate code:" << code << ", " << translateCode << ", msg:" << errMagBuf_ << endl; - return (code == translateCode); + translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; + return (terrno == translateCode); } - cout << "input sql : [" << cxt_.pSql << "]" << endl; - cout << "parser output: " << endl; - cout << parserStr << endl; - cout << "translate output: " << endl; - cout << toString(query_->pRoot) << endl; + translatedAstStr_ = toString(query_->pRoot); return (TSDB_CODE_SUCCESS == translateCode); } + void dump() { + cout << "input sql : [" << cxt_.pSql << "]" << endl; + if (!parseErrStr_.empty()) { + cout << "parse error: " << parseErrStr_ << endl; + } + if (!parsedAstStr_.empty()) { + cout << "parse output: " << endl; + cout << parsedAstStr_ << endl; + } + if (!translateErrStr_.empty()) { + cout << "translate error: " << translateErrStr_ << endl; + } + if (!translatedAstStr_.empty()) { + cout << "translate output: " << endl; + cout << translatedAstStr_ << endl; + } + } + string toString(const SNode* pRoot, bool format = false) { char* pStr = NULL; int32_t len = 0; @@ -82,7 +99,7 @@ private: throw "nodesNodeToString failed!"; } string str(pStr); - tfree(pStr); + taosMemoryFreeClear(pStr); return str; } @@ -91,6 +108,10 @@ private: memset(errMagBuf_, 0, max_err_len); cxt_.pMsg = errMagBuf_; cxt_.msgLen = max_err_len; + parseErrStr_.clear(); + parsedAstStr_.clear(); + translateErrStr_.clear(); + translatedAstStr_.clear(); } string acctId_; @@ -99,8 +120,50 @@ private: string sqlBuf_; SParseContext cxt_; SQuery* query_; + string parseErrStr_; + string parsedAstStr_; + string translateErrStr_; + string translatedAstStr_; }; +TEST_F(ParserTest, createAccount) { + setDatabase("root", "test"); + + bind("create account ac_wxy pass '123456'"); + ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); +} + +TEST_F(ParserTest, alterAccount) { + setDatabase("root", "test"); + + bind("alter account ac_wxy pass '123456'"); + ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); +} + +TEST_F(ParserTest, createUser) { + setDatabase("root", "test"); + + bind("create user wxy pass '123456'"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, alterUser) { + setDatabase("root", "test"); + + bind("alter user wxy pass '123456'"); + ASSERT_TRUE(run()); + + bind("alter user wxy privilege 'write'"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, dropUser) { + setDatabase("root", "test"); + + bind("drop user wxy"); + ASSERT_TRUE(run()); +} + TEST_F(ParserTest, selectSimple) { setDatabase("root", "test"); @@ -295,20 +358,13 @@ TEST_F(ParserTest, selectSemanticError) { ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); } -TEST_F(ParserTest, createUser) { +TEST_F(ParserTest, showUsers) { setDatabase("root", "test"); - bind("create user wxy pass '123456'"); + bind("show users"); ASSERT_TRUE(run()); } -TEST_F(ParserTest, alterAccount) { - setDatabase("root", "test"); - - bind("alter account ac_wxy pass '123456'"); - ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); -} - TEST_F(ParserTest, createDnode) { setDatabase("root", "test"); @@ -319,6 +375,13 @@ TEST_F(ParserTest, createDnode) { ASSERT_TRUE(run()); } +TEST_F(ParserTest, showDnodes) { + setDatabase("root", "test"); + + bind("show dnodes"); + ASSERT_TRUE(run()); +} + TEST_F(ParserTest, alterDnode) { setDatabase("root", "test"); @@ -433,6 +496,93 @@ TEST_F(ParserTest, createTable) { ASSERT_TRUE(run()); } +TEST_F(ParserTest, showTables) { + setDatabase("root", "test"); + + bind("show tables"); + ASSERT_TRUE(run()); + + bind("show test.tables"); + ASSERT_TRUE(run()); + + bind("show tables like 'c%'"); + ASSERT_TRUE(run()); + + bind("show test.tables like 'c%'"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showStables) { + setDatabase("root", "test"); + + bind("show stables"); + ASSERT_TRUE(run()); + + bind("show test.stables"); + ASSERT_TRUE(run()); + + bind("show stables like 'c%'"); + ASSERT_TRUE(run()); + + bind("show test.stables like 'c%'"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showVgroups) { + setDatabase("root", "test"); + + bind("show vgroups"); + ASSERT_TRUE(run()); + + bind("show test.vgroups"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showMnodes) { + setDatabase("root", "test"); + + bind("show mnodes"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showModules) { + setDatabase("root", "test"); + + bind("show modules"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showQnodes) { + setDatabase("root", "test"); + + bind("show qnodes"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showFunctions) { + setDatabase("root", "test"); + + bind("show functions"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showIndexes) { + setDatabase("root", "test"); + + bind("show indexes from t1"); + ASSERT_TRUE(run()); + + bind("show indexes from t1 from test"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showStreams) { + setDatabase("root", "test"); + + bind("show streams"); + ASSERT_TRUE(run()); +} + TEST_F(ParserTest, createSmaIndex) { setDatabase("root", "test"); diff --git a/source/libs/parser/test/parserInsertTest.cpp b/source/libs/parser/test/parserInsertTest.cpp index 3d4a6e0eb86b0901b555aae3e21e668ccfcd30ce..90e2ba3db2d18e8552078a5647c75525131b8211 100644 --- a/source/libs/parser/test/parserInsertTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -72,7 +72,7 @@ protected: SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); for (int32_t i = 0; i < numOfBlocks; ++i) { cout << "Block:" << i << endl; - cout << "\tuid:" << be64toh(blk->uid) << ", tid:" << ntohl(blk->tid) << ", padding:" << ntohl(blk->padding) << ", sversion:" << ntohl(blk->sversion) + cout << "\tuid:" << be64toh(blk->uid) << ", tid:" << be64toh(blk->suid) << ", padding:" << ntohl(blk->padding) << ", sversion:" << ntohl(blk->sversion) << ", dataLen:" << ntohl(blk->dataLen) << ", schemaLen:" << ntohl(blk->schemaLen) << ", numOfRows:" << ntohs(blk->numOfRows) << endl; blk = (SSubmitBlk*)(blk->data + ntohl(blk->dataLen)); } @@ -131,7 +131,7 @@ private: TEST_F(InsertTest, singleTableSingleRowTest) { setDatabase("root", "test"); - bind("insert into t1 values (now, 1, \"beijing\")"); + bind("insert into t1 values (now, 1, 'beijing', 3, 4, 5)"); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); dumpReslut(); checkReslut(1, 1); @@ -141,7 +141,7 @@ TEST_F(InsertTest, singleTableSingleRowTest) { TEST_F(InsertTest, singleTableMultiRowTest) { setDatabase("root", "test"); - bind("insert into t1 values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); + bind("insert into t1 values (now, 1, 'beijing', 3, 4, 5)(now+1s, 2, 'shanghai', 6, 7, 8)(now+2s, 3, 'guangzhou', 9, 10, 11)"); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); dumpReslut(); checkReslut(1, 3); diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index be271cf05775ab7bdac5fd393ada78151621c5f6..42449d63d6c2d79ffe586f2d5ee1d96567d3bc85 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -56,9 +56,10 @@ extern "C" { #define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); -int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode); -int32_t applySplitRule(SSubLogicPlan* pSubplan); -int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan, SArray* pExecNodeList); +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode); +int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan); +int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan); +int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList); #ifdef __cplusplus } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index cbc3921711cd15b224b37ed68c823978ab89e21e..6ea476a334a104e329e64d14ef131ee069d34662 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -123,6 +123,30 @@ static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelec return code; } +static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanCols, STableMeta* pMeta) { + if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { + return SCAN_TYPE_STREAM; + } + + if (NULL == pScanCols) { + // select count(*) from t + return SCAN_TYPE_TABLE; + } + + if (TSDB_SYSTEM_TABLE == pMeta->tableType) { + return SCAN_TYPE_SYSTEM_TABLE; + } + + SNode* pCol = NULL; + FOREACH(pCol, pScanCols) { + if (COLUMN_TYPE_COLUMN == ((SColumnNode*)pCol)->colType) { + return SCAN_TYPE_TABLE; + } + } + + return SCAN_TYPE_TAG; +} + static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) { SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); if (NULL == pScan) { @@ -131,13 +155,13 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*); - pScan->scanType = pCxt->pPlanCxt->streamQuery ? SCAN_TYPE_STREAM : SCAN_TYPE_TABLE; pScan->scanFlag = MAIN_SCAN; pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; pScan->tableName.acctId = pCxt->pPlanCxt->acctId; strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); + pScan->showRewrite = pCxt->pPlanCxt->showRewrite; // set columns to scan SNodeList* pCols = NULL; @@ -149,6 +173,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } } + pScan->scanType = getScanType(pCxt, pCols, pScan->pMeta); + // set output if (TSDB_CODE_SUCCESS == code && NULL != pCols) { pScan->node.pTargets = nodesCloneList(pCols); @@ -386,6 +412,38 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, return code; } +static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { + int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs); + + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); + } + + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pWindow; + } else { + nodesDestroyNode(pWindow); + } + + return code; +} + +static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionWindowNode* pSession, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); + if (NULL == pWindow) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pWindow->winType = WINDOW_TYPE_SESSION; + pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; + + return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); +} + static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); if (NULL == pWindow) { @@ -399,34 +457,15 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); - int32_t code = TSDB_CODE_SUCCESS; - if (NULL != pInterval->pFill) { pWindow->pFill = nodesCloneNode(pInterval->pFill); if (NULL == pWindow->pFill) { - code = TSDB_CODE_OUT_OF_MEMORY; + nodesDestroyNode(pWindow); + return TSDB_CODE_OUT_OF_MEMORY; } } - if (TSDB_CODE_SUCCESS == code) { - code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs); - } - - if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); - } - - if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets); - } - - if (TSDB_CODE_SUCCESS == code) { - *pLogicNode = (SLogicNode*)pWindow; - } else { - nodesDestroyNode(pWindow); - } - - return code; + return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { @@ -435,8 +474,10 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele } switch (nodeType(pSelect->pWindow)) { + case QUERY_NODE_SESSION_WINDOW: + return createWindowLogicNodeBySession(pCxt, (SSessionWindowNode*)pSelect->pWindow, pSelect, pLogicNode); case QUERY_NODE_INTERVAL_WINDOW: - return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); + return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); default: break; } diff --git a/source/client/stream/stream.c b/source/libs/planner/src/planOptimizer.c similarity index 82% rename from source/client/stream/stream.c rename to source/libs/planner/src/planOptimizer.c index 6dea4a4e57392be988126c579648f39a8270b9bf..2a36e38ce1f7f84de807a4f2eeee1229fcdc41ec 100644 --- a/source/client/stream/stream.c +++ b/source/libs/planner/src/planOptimizer.c @@ -11,4 +11,10 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "planInt.h" + +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 9bd926b67634f94d876756df0c37fa943d11a6e0..d8d090d280dc21f7285c50ca8e7c6943a07e3f48 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -28,7 +28,6 @@ typedef struct SPhysiPlanContext { int16_t nextDataBlockId; SArray* pLocationHelper; SArray* pExecNodeList; - int32_t subplanId; } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, char* pKey) { @@ -124,35 +123,52 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNode* pNode) { +static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNode* pNode, SNode** pOutput) { SNode* pRes = nodesCloneNode(pNode); - CHECK_ALLOC(pRes, NULL); - SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), - .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; + if (NULL == pRes) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSetSlotIdCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, + .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) + }; nodesWalkNode(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); - return NULL; + return cxt.errCode; } - return pRes; + + *pOutput = pRes; + return TSDB_CODE_SUCCESS; } -static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNodeList* pList) { +static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNodeList* pList, SNodeList** pOutput) { SNodeList* pRes = nodesCloneList(pList); - CHECK_ALLOC(pRes, NULL); - SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), - .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; + if (NULL == pRes) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSetSlotIdCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, + .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) + }; nodesWalkList(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); - return NULL; + return cxt.errCode; } - return pRes; + *pOutput = pRes; + return TSDB_CODE_SUCCESS; } static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); - CHECK_ALLOC(pPhysiNode, NULL); + if (NULL == pPhysiNode) { + return NULL; + } pPhysiNode->pOutputDataBlockDesc = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC); if (NULL == pPhysiNode->pOutputDataBlockDesc) { nodesDestroyNode(pPhysiNode); @@ -165,8 +181,7 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) { if (NULL != pLogicNode->pConditions) { - pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->pOutputDataBlockDesc->dataBlockId, -1, pLogicNode->pConditions); - CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY); + return setNodeSlotId(pCxt, pPhysiNode->pOutputDataBlockDesc->dataBlockId, -1, pLogicNode->pConditions, &pPhysiNode->pConditions); } return TSDB_CODE_SUCCESS; } @@ -198,42 +213,87 @@ static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) { return pCol; } -static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) { - pScanPhysiNode->pScanCols = nodesMakeList(); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))); +static int32_t colIdCompare(const void* pLeft, const void* pRight) { + SColumnNode* pLeftCol = *(SColumnNode**)pLeft; + SColumnNode* pRightCol = *(SColumnNode**)pRight; + return pLeftCol->colId > pRightCol->colId ? 1 : -1; +} - SNode* pNode; - FOREACH(pNode, pScanCols) { - if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) { - SColumnNode* pCol = nodesListGetNode(pScanPhysiNode->pScanCols, 0); - strcpy(pCol->tableAlias, ((SColumnNode*)pNode)->tableAlias); - strcpy(pCol->colName, ((SColumnNode*)pNode)->colName); - continue; - } - CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))); +static int32_t sortScanCols(SNodeList* pScanCols) { + SArray* pArray = taosArrayInit(LIST_LENGTH(pScanCols), POINTER_BYTES); + if (NULL == pArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNode* pCol = NULL; + FOREACH(pCol, pScanCols) { + taosArrayPush(pArray, &pCol); } + taosArraySort(pArray, colIdCompare); + + int32_t index = 0; + FOREACH(pCol, pScanCols) { + REPLACE_NODE(taosArrayGetP(pArray, index++)); + } + taosArrayDestroy(pArray); + return TSDB_CODE_SUCCESS; } -static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { - CHECK_CODE(createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); - - // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t - CHECK_CODE(addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY); +static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) { + if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanPhysiNode) + || QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN == nodeType(pScanPhysiNode)) { + pScanPhysiNode->pScanCols = nodesMakeList(); + CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))); + + SNode* pNode; + FOREACH(pNode, pScanCols) { + if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) { + SColumnNode* pCol = nodesListGetNode(pScanPhysiNode->pScanCols, 0); + strcpy(pCol->tableAlias, ((SColumnNode*)pNode)->tableAlias); + strcpy(pCol->colName, ((SColumnNode*)pNode)->colName); + continue; + } + CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))); + } + } else { + pScanPhysiNode->pScanCols = nodesCloneList(pScanCols); + CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); + } - CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY); + // return sortScanCols(pScanPhysiNode->pScanCols); + return TSDB_CODE_SUCCESS; +} - CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, pScanPhysiNode->node.pOutputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY); +static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode, SPhysiNode** pPhyNode) { + int32_t code = createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols); + if (TSDB_CODE_SUCCESS == code) { + // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t + code = addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = setSlotOutput(pCxt, pScanLogicNode->node.pTargets, pScanPhysiNode->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; + pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; + pScanPhysiNode->order = TSDB_ORDER_ASC; + pScanPhysiNode->count = 1; + pScanPhysiNode->reverse = 0; + memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); + } - pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; - pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; - pScanPhysiNode->order = TSDB_ORDER_ASC; - pScanPhysiNode->count = 1; - pScanPhysiNode->reverse = 0; - memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pScanPhysiNode; + } else { + nodesDestroyNode(pScanPhysiNode); + } - return TSDB_CODE_SUCCESS; + return code; } static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) { @@ -241,101 +301,145 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd pNodeAddr->epSet = vg->epSet; } -static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { +static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); - CHECK_ALLOC(pTagScan, NULL); - CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan); - return (SPhysiNode*)pTagScan; + if (NULL == pTagScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode); } -static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { +static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); - CHECK_ALLOC(pTableScan, NULL); - CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); + if (NULL == pTableScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pTableScan->scanFlag = pScanLogicNode->scanFlag; pTableScan->scanRange = pScanLogicNode->scanRange; vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); - return (SPhysiNode*)pTableScan; + pSubplan->execNodeStat.tableNum = pScanLogicNode->pVgroupList->vgroups[0].numOfTable; + tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); + + return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); +} + +static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pScan->showRewrite = pScanLogicNode->showRewrite; + pScan->accountId = pCxt->pPlanCxt->acctId; + if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES)) { + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); + } else { + SQueryNodeAddr addr = { .nodeId = MND_VGID, .epSet = pCxt->pPlanCxt->mgmtEpSet }; + taosArrayPush(pCxt->pExecNodeList, &addr); + } + pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; + tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); + + return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static SPhysiNode* createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { - SStreamScanPhysiNode* pTableScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); - CHECK_ALLOC(pTableScan, NULL); - CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); - return (SPhysiNode*)pTableScan; +static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { +static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: - return createTagScanPhysiNode(pCxt, pScanLogicNode); + return createTagScanPhysiNode(pCxt, pScanLogicNode, pPhyNode); case SCAN_TYPE_TABLE: - return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode); + return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); + case SCAN_TYPE_SYSTEM_TABLE: + return createSystemTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_STREAM: - return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode); + return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); default: break; } - return NULL; + return TSDB_CODE_FAILED; } -static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, SDataBlockDescNode* pLeftDesc, SDataBlockDescNode* pRightDesc) { - SNodeList* pCols = nodesMakeList(); - CHECK_ALLOC(pCols, NULL); +static int32_t createColFromDataBlockDesc(SDataBlockDescNode* pDesc, SNodeList* pCols) { SNode* pNode; - FOREACH(pNode, pLeftDesc->pSlots) { + FOREACH(pNode, pDesc->pSlots) { SSlotDescNode* pSlot = (SSlotDescNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { - goto error; + return TSDB_CODE_OUT_OF_MEMORY; } pCol->node.resType = pSlot->dataType; - pCol->dataBlockId = pLeftDesc->dataBlockId; + pCol->dataBlockId = pDesc->dataBlockId; pCol->slotId = pSlot->slotId; pCol->colId = -1; - if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { - goto error; + int32_t code = nodesListStrictAppend(pCols, pCol); + if (TSDB_CODE_SUCCESS != code) { + return code; } } - FOREACH(pNode, pRightDesc->pSlots) { - SSlotDescNode* pSlot = (SSlotDescNode*)pNode; - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - goto error; - } - pCol->node.resType = pSlot->dataType; - pCol->dataBlockId = pRightDesc->dataBlockId; - pCol->slotId = pSlot->slotId; - pCol->colId = -1; - if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { - goto error; - } + return TSDB_CODE_SUCCESS; +} + +static int32_t createJoinOutputCols(SPhysiPlanContext* pCxt, SDataBlockDescNode* pLeftDesc, SDataBlockDescNode* pRightDesc, SNodeList** pList) { + SNodeList* pCols = nodesMakeList(); + if (NULL == pCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = createColFromDataBlockDesc(pLeftDesc, pCols); + if (TSDB_CODE_SUCCESS == code) { + code = createColFromDataBlockDesc(pRightDesc, pCols); } - return pCols; -error: - nodesDestroyList(pCols); - return NULL; + + if (TSDB_CODE_SUCCESS == code) { + *pList = pCols; + } else { + nodesDestroyList(pCols); + } + + return code; } -static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) { +static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode, SPhysiNode** pPhyNode) { SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN); - CHECK_ALLOC(pJoin, NULL); + if (NULL == pJoin) { + return TSDB_CODE_OUT_OF_MEMORY; + } SDataBlockDescNode* pLeftDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc; SDataBlockDescNode* pRightDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc; - pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions); - CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin); - - pJoin->pTargets = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc); - CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin); - CHECK_CODE(addDataBlockDesc(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc), (SPhysiNode*)pJoin); - CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin); + int32_t code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); + if (TSDB_CODE_SUCCESS == code) { + code = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc, &pJoin->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); + } + if (TSDB_CODE_SUCCESS == code) { + code = setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, pJoin->node.pOutputDataBlockDesc); + } - CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, pJoin->node.pOutputDataBlockDesc), (SPhysiNode*)pJoin); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pJoin; + } else { + nodesDestroyNode(pJoin); + } - return (SPhysiNode*)pJoin; + return code; } typedef struct SRewritePrecalcExprsCxt { @@ -421,67 +525,169 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN return cxt.errCode; } -static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) { +static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) { SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG); - CHECK_ALLOC(pAgg, NULL); + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } SNodeList* pPrecalcExprs = NULL; SNodeList* pGroupKeys = NULL; SNodeList* pAggFuncs = NULL; - CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg); - CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg); + int32_t code = rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys); + if (TSDB_CODE_SUCCESS == code) { + code = rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs); + } SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); // push down expression to pOutputDataBlockDesc of child node - if (NULL != pPrecalcExprs) { - pAgg->pExprs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs); - CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg); - CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pAgg->pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pAgg->pExprs, pChildTupe); + } } - if (NULL != pGroupKeys) { - pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pGroupKeys); - CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg); - CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pGroupKeys, &pAgg->pGroupKeys); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc); + } } - if (NULL != pAggFuncs) { - pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pAggFuncs); - CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg); - CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pAggFuncs, &pAgg->pAggFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc); + } } - CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg); + } + if (TSDB_CODE_SUCCESS == code) { + code = setSlotOutput(pCxt, pAggLogicNode->node.pTargets, pAgg->node.pOutputDataBlockDesc); + } - CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } - return (SPhysiNode*)pAgg; + return code; } -static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) { +static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT); - CHECK_ALLOC(pProject, NULL); + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } - pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections); - CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject); - CHECK_CODE(addDataBlockDesc(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc), (SPhysiNode*)pProject); + int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections, &pProject->pProjections); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject); + } - CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pProject; + } else { + nodesDestroyNode(pProject); + } - return (SPhysiNode*)pProject; + return code; } -static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) { +static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); - CHECK_ALLOC(pExchange, NULL); - CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; - return (SPhysiNode*)pExchange; + int32_t code = addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc); + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pExchange; + } else { + nodesDestroyNode(pExchange); + } + + return code; +} +static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc); + if (TSDB_CODE_SUCCESS == code) { + pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets); + if (NULL == pScan->pScanCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pScan; + } else { + nodesDestroyNode(pScan); + } + + return code; } -static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) { +static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { + if (pCxt->pPlanCxt->streamQuery) { + return createStreamScanPhysiNodeByExchange(pCxt, pExchangeLogicNode, pPhyNode); + } else { + return doCreateExchangePhysiNode(pCxt, pExchangeLogicNode, pPhyNode); + } +} + +static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { + SNodeList* pPrecalcExprs = NULL; + SNodeList* pFuncs = NULL; + int32_t code = rewritePrecalcExprs(pCxt, pWindowLogicNode->pFuncs, &pPrecalcExprs, &pFuncs); + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + // push down expression to pOutputDataBlockDesc of child node + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pWindow->pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pWindow->pExprs, pChildTupe); + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pFuncs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs, &pWindow->pFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pWindow->pFuncs, pWindow->node.pOutputDataBlockDesc); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = setSlotOutput(pCxt, pWindowLogicNode->node.pTargets, pWindow->node.pOutputDataBlockDesc); + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pWindow; + } else { + nodesDestroyNode(pWindow); + } + + return code; +} + +static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_INTERVAL); - CHECK_ALLOC(pInterval, NULL); + if (NULL == pInterval) { + return TSDB_CODE_OUT_OF_MEMORY; + } pInterval->interval = pWindowLogicNode->interval; pInterval->offset = pWindowLogicNode->offset; @@ -490,161 +696,178 @@ static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* p pInterval->slidingUnit = pWindowLogicNode->slidingUnit; pInterval->pFill = nodesCloneNode(pWindowLogicNode->pFill); - - SNodeList* pPrecalcExprs = NULL; - SNodeList* pFuncs = NULL; - CHECK_CODE(rewritePrecalcExprs(pCxt, pWindowLogicNode->pFuncs, &pPrecalcExprs, &pFuncs), (SPhysiNode*)pInterval); - - SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); - // push down expression to pOutputDataBlockDesc of child node - if (NULL != pPrecalcExprs) { - pInterval->pExprs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs); - CHECK_ALLOC(pInterval->pExprs, (SPhysiNode*)pInterval); - CHECK_CODE(addDataBlockDesc(pCxt, pInterval->pExprs, pChildTupe), (SPhysiNode*)pInterval); + if (NULL != pWindowLogicNode->pFill && NULL == pInterval->pFill) { + nodesDestroyNode(pInterval); + return TSDB_CODE_OUT_OF_MEMORY; } - if (NULL != pFuncs) { - pInterval->pFuncs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs); - CHECK_ALLOC(pInterval->pFuncs, (SPhysiNode*)pInterval); - CHECK_CODE(addDataBlockDesc(pCxt, pInterval->pFuncs, pInterval->node.pOutputDataBlockDesc), (SPhysiNode*)pInterval); + return createWindowPhysiNodeFinalize(pCxt, pChildren, &pInterval->window, pWindowLogicNode, pPhyNode); +} + +static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { + SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW); + if (NULL == pSession) { + return TSDB_CODE_OUT_OF_MEMORY; } - CHECK_CODE(setSlotOutput(pCxt, pWindowLogicNode->node.pTargets, pInterval->node.pOutputDataBlockDesc), (SPhysiNode*)pInterval); + pSession->gap = pWindowLogicNode->sessionGap; - return (SPhysiNode*)pInterval; + return createWindowPhysiNodeFinalize(pCxt, pChildren, &pSession->window, pWindowLogicNode, pPhyNode); } -static SPhysiNode* createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) { +static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { switch (pWindowLogicNode->winType) { case WINDOW_TYPE_INTERVAL: - return createIntervalPhysiNode(pCxt, pChildren, pWindowLogicNode); + return createIntervalPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); case WINDOW_TYPE_SESSION: + return createSessionWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); case WINDOW_TYPE_STATE: break; default: break; } - return NULL; + return TSDB_CODE_FAILED; } -static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SLogicNode* pLogicPlan) { - SNodeList* pChildren = nodesMakeList(); - CHECK_ALLOC(pChildren, NULL); - - SNode* pLogicChild; - FOREACH(pLogicChild, pLogicPlan->pChildren) { - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pChildren, createPhysiNode(pCxt, pSubplan, (SLogicNode*)pLogicChild))) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - nodesDestroyList(pChildren); - return NULL; - } - } - - SPhysiNode* pPhyNode = NULL; - switch (nodeType(pLogicPlan)) { +static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { + switch (nodeType(pLogicNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - pPhyNode = createScanPhysiNode(pCxt, pSubplan, (SScanLogicNode*)pLogicPlan); - break; + return createScanPhysiNode(pCxt, pSubplan, (SScanLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_JOIN: - pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan); - break; + return createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_AGG: - pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan); - break; + return createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_PROJECT: - pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan); - break; + return createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: - pPhyNode = createExchangePhysiNode(pCxt, (SExchangeLogicNode*)pLogicPlan); - break; + return createExchangePhysiNode(pCxt, (SExchangeLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: - pPhyNode = createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicPlan); - break; + return createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicNode, pPhyNode); default: break; } - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode(pPhyNode); - return NULL; + + return TSDB_CODE_FAILED; +} + +static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SPhysiNode** pPhyNode) { + SNodeList* pChildren = nodesMakeList(); + if (NULL == pChildren) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + + SNode* pLogicChild; + FOREACH(pLogicChild, pLogicNode->pChildren) { + SPhysiNode* pChild = NULL; + code = createPhysiNode(pCxt, (SLogicNode*)pLogicChild, pSubplan, &pChild); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pChildren, pChild); + } } - pPhyNode->pChildren = pChildren; - SNode* pChild; - FOREACH(pChild, pPhyNode->pChildren) { - ((SPhysiNode*)pChild)->pParent = pPhyNode; + if (TSDB_CODE_SUCCESS == code) { + code = doCreatePhysiNode(pCxt, pLogicNode, pSubplan, pChildren, pPhyNode); } - return pPhyNode; + if (TSDB_CODE_SUCCESS == code) { + (*pPhyNode)->pChildren = pChildren; + SNode* pChild; + FOREACH(pChild, (*pPhyNode)->pChildren) { + ((SPhysiNode*)pChild)->pParent = (*pPhyNode); + } + } else { + nodesDestroyList(pChildren); + } + + return code; } -static SDataSinkNode* createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlocks) { +static int32_t createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlocks, SDataSinkNode** pSink) { SDataInserterNode* pInserter = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); - CHECK_ALLOC(pInserter, NULL); + if (NULL == pInserter) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pInserter->numOfTables = pBlocks->numOfTables; pInserter->size = pBlocks->size; TSWAP(pInserter->pData, pBlocks->pData, char*); - return (SDataSinkNode*)pInserter; + + *pSink = (SDataSinkNode*)pInserter; + return TSDB_CODE_SUCCESS; } -static SDataSinkNode* createDataDispatcher(SPhysiPlanContext* pCxt, const SPhysiNode* pRoot) { +static int32_t createDataDispatcher(SPhysiPlanContext* pCxt, const SPhysiNode* pRoot, SDataSinkNode** pSink) { SDataDispatcherNode* pDispatcher = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DISPATCH); - CHECK_ALLOC(pDispatcher, NULL); + if (NULL == pDispatcher) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pDispatcher->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc); - CHECK_ALLOC(pDispatcher->sink.pInputDataBlockDesc, (SDataSinkNode*)pDispatcher); - return (SDataSinkNode*)pDispatcher; + if (NULL == pDispatcher->sink.pInputDataBlockDesc) { + nodesDestroyNode(pDispatcher); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pSink = (SDataSinkNode*)pDispatcher; + return TSDB_CODE_SUCCESS; } -static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) { +static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { SSubplan* pSubplan = nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); - CHECK_ALLOC(pSubplan, NULL); + if (NULL == pSubplan) { + return NULL; + } pSubplan->id = pLogicSubplan->id; pSubplan->subplanType = pLogicSubplan->subplanType; pSubplan->level = pLogicSubplan->level; return pSubplan; } -static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) { +static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan** pPhysiSubplan) { SSubplan* pSubplan = makeSubplan(pCxt, pLogicSubplan); - CHECK_ALLOC(pSubplan, NULL); + if (NULL == pSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + if (SUBPLAN_TYPE_MODIFY == pLogicSubplan->subplanType) { SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode; - pSubplan->pDataSink = createDataInserter(pCxt, pModif->pVgDataBlocks); pSubplan->msgType = pModif->msgType; pSubplan->execNode.epSet = pModif->pVgDataBlocks->vg.epSet; taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); + code = createDataInserter(pCxt, pModif->pVgDataBlocks, &pSubplan->pDataSink); } else { - pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode); - pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); pSubplan->msgType = TDMT_VND_QUERY; + code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); + if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { + code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); + } } - return pSubplan; -} -static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { - pNode->pParent = pParent; - SNode* pChild; - FOREACH(pChild, pNode->pChildren) { - doSetLogicNodeParent((SLogicNode*)pChild, pNode); + if (TSDB_CODE_SUCCESS == code) { + *pPhysiSubplan = pSubplan; + } else { + nodesDestroyNode(pSubplan); } + + return code; } -static void setLogicNodeParent(SLogicNode* pNode) { - doSetLogicNodeParent(pNode, NULL); -} - -static int32_t splitLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubLogicPlan** pSubLogicPlan) { - *pSubLogicPlan = (SSubLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); - CHECK_ALLOC(*pSubLogicPlan, TSDB_CODE_OUT_OF_MEMORY); - (*pSubLogicPlan)->pNode = nodesCloneNode(pLogicNode); - if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) { - (*pSubLogicPlan)->subplanType = SUBPLAN_TYPE_MODIFY; - TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)(*pSubLogicPlan)->pNode)->pDataBlocks, SArray*); - } else { - (*pSubLogicPlan)->subplanType = SUBPLAN_TYPE_SCAN; +static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) { + SQueryPlan* pPlan = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN); + if (NULL == pPlan) { + return NULL; } - (*pSubLogicPlan)->id.queryId = pCxt->pPlanCxt->queryId; - setLogicNodeParent((*pSubLogicPlan)->pNode); - return applySplitRule(*pSubLogicPlan); + pPlan->pSubplans = nodesMakeList(); + if (NULL == pPlan->pSubplans) { + nodesDestroyNode(pPlan); + return NULL; + } + pPlan->queryId = pCxt->pPlanCxt->queryId; + return pPlan; } static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t level, SNodeList* pSubplans) { @@ -664,193 +887,65 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t l return TSDB_CODE_SUCCESS; } -static SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSrc, int32_t level) { - SSubLogicPlan* pDst = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); - CHECK_ALLOC(pDst, NULL); - pDst->pNode = nodesCloneNode(pSrc->pNode); - if (NULL == pDst->pNode) { - nodesDestroyNode(pDst); - return NULL; - } - pDst->subplanType = pSrc->subplanType; - pDst->level = level; - pDst->id.queryId = pSrc->id.queryId; - pDst->id.groupId = pSrc->id.groupId; - pDst->id.subplanId = pCxt->subplanId++; - return pDst; -} - -static int32_t scaleOutForModify(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { - SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; - size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); - for (int32_t i = 0; i < numOfVgroups; ++i) { - SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); - ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = blocks; - CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t scaleOutForMerge(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { - return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); -} - -static int32_t doSetScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - SScanLogicNode* pScan = (SScanLogicNode*)pNode; - pScan->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); - CHECK_ALLOC(pScan->pVgroupList, TSDB_CODE_OUT_OF_MEMORY); - memcpy(pScan->pVgroupList->vgroups, pVgroup, sizeof(SVgroupInfo)); - *pFound = true; - return TSDB_CODE_SUCCESS; - } - SNode* pChild = NULL; - FOREACH(pChild, pNode->pChildren) { - int32_t code = doSetScanVgroup(pCxt, (SLogicNode*)pChild, pVgroup, pFound); - if (TSDB_CODE_SUCCESS != code || *pFound) { - return code; - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t setScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const SVgroupInfo* pVgroup) { - bool found = false; - return doSetScanVgroup(pCxt, pNode, pVgroup, &found); -} +static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent, SQueryPlan* pQueryPlan) { + SSubplan* pSubplan = NULL; + int32_t code = createPhysiSubplan(pCxt, pLogicSubplan, &pSubplan); -static int32_t scaleOutForScan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { - if (pSubplan->pVgroupList) { - for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { - SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(setScanVgroup(pCxt, pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i)); - CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); - } - return TSDB_CODE_SUCCESS; - } else { - return scaleOutForMerge(pCxt, pSubplan, level, pGroup); + if (TSDB_CODE_SUCCESS == code) { + code = pushSubplan(pCxt, pSubplan, pLogicSubplan->level, pQueryPlan->pSubplans); + ++(pQueryPlan->numOfSubplans); } -} -static int32_t appendWithMakeList(SNodeList** pList, SNodeptr pNode) { - if (NULL == *pList) { - *pList = nodesMakeList(); - if (NULL == *pList) { - return TSDB_CODE_OUT_OF_MEMORY; + if (TSDB_CODE_SUCCESS == code && NULL != pParent) { + code = nodesListMakeAppend(&pParent->pChildren, pSubplan); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pSubplan->pParents, pParent); } } - return nodesListAppend(*pList, pNode); -} -static int32_t pushHierarchicalPlan(SPhysiPlanContext* pCxt, SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { - bool topLevel = (0 == LIST_LENGTH(pParentsGroup)); - SNode* pChild = NULL; - FOREACH(pChild, pCurrentGroup) { - if (topLevel) { - CHECK_CODE_EXT(nodesListAppend(pParentsGroup, pChild)); - } else { - SNode* pParent = NULL; - FOREACH(pParent, pParentsGroup) { - CHECK_CODE_EXT(appendWithMakeList(&(((SSubLogicPlan*)pParent)->pChildren), pChild)); - CHECK_CODE_EXT(appendWithMakeList(&(((SSubLogicPlan*)pChild)->pParents), pParent)); + if (TSDB_CODE_SUCCESS == code) { + SNode* pChild = NULL; + FOREACH(pChild, pLogicSubplan->pChildren) { + code = buildPhysiPlan(pCxt, (SLogicSubplan*)pChild, pSubplan, pQueryPlan); + if (TSDB_CODE_SUCCESS != code) { + break; } } } - return TSDB_CODE_SUCCESS; -} -static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t* pLevel, SNodeList* pParentsGroup) { - SNodeList* pCurrentGroup = nodesMakeList(); - CHECK_ALLOC(pCurrentGroup, TSDB_CODE_OUT_OF_MEMORY); - int32_t code = TSDB_CODE_SUCCESS; - switch (pSubplan->subplanType) { - case SUBPLAN_TYPE_MERGE: - code = scaleOutForMerge(pCxt, pSubplan, *pLevel, pCurrentGroup); - break; - case SUBPLAN_TYPE_SCAN: - code = scaleOutForScan(pCxt, pSubplan, *pLevel, pCurrentGroup); - break; - case SUBPLAN_TYPE_MODIFY: - code = scaleOutForModify(pCxt, pSubplan, *pLevel, pCurrentGroup); - break; - default: - break; - } if (TSDB_CODE_SUCCESS != code) { - return code; - } - - CHECK_CODE_EXT(pushHierarchicalPlan(pCxt, pParentsGroup, pCurrentGroup)); - ++(*pLevel); - SNode* pChild; - FOREACH(pChild, pSubplan->pChildren) { - CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, pLevel, pCurrentGroup)); + nodesDestroyNode(pSubplan); } - return TSDB_CODE_SUCCESS; + return code; } -static SQueryLogicPlan* makeQueryLogicPlan(SPhysiPlanContext* pCxt) { - SQueryLogicPlan* pLogicPlan = (SQueryLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN); - CHECK_ALLOC(pLogicPlan, NULL); - pLogicPlan->pTopSubplans = nodesMakeList(); - if (NULL == pLogicPlan->pTopSubplans) { - nodesDestroyNode(pLogicPlan); - return NULL; +static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPhysiPlan) { + SQueryPlan* pPlan = makeQueryPhysiPlan(pCxt); + if (NULL == pPlan) { + return TSDB_CODE_OUT_OF_MEMORY; } - return pLogicPlan; -} -static int32_t scaleOutLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pRootSubLogicPlan, SQueryLogicPlan** pLogicPlan) { - *pLogicPlan = makeQueryLogicPlan(pCxt); - CHECK_ALLOC(*pLogicPlan, TSDB_CODE_OUT_OF_MEMORY); - return doScaleOut(pCxt, pRootSubLogicPlan, &((*pLogicPlan)->totalLevel), (*pLogicPlan)->pTopSubplans); -} - -static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) { - SQueryPlan* pPlan = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN); - CHECK_ALLOC(pPlan, NULL); - pPlan->pSubplans = nodesMakeList(); - if (NULL == pPlan->pSubplans) { - nodesDestroyNode(pPlan); - return NULL; - } - pPlan->queryId = pCxt->pPlanCxt->queryId; - return pPlan; -} + int32_t code = TSDB_CODE_SUCCESS; -static int32_t doBuildPhysiPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan, SSubplan* pParent, SQueryPlan* pQueryPlan) { - SSubplan* pSubplan = createPhysiSubplan(pCxt, pLogicSubplan); - CHECK_ALLOC(pSubplan, DEAL_RES_ERROR); - CHECK_CODE_EXT(pushSubplan(pCxt, pSubplan, pLogicSubplan->level, pQueryPlan->pSubplans)); - ++(pQueryPlan->numOfSubplans); - if (NULL != pParent) { - CHECK_CODE_EXT(appendWithMakeList(&pParent->pChildren, pSubplan)); - CHECK_CODE_EXT(appendWithMakeList(&pSubplan->pParents, pParent)); + SNode* pSubplan = NULL; + FOREACH(pSubplan, pLogicPlan->pTopSubplans) { + code = buildPhysiPlan(pCxt, (SLogicSubplan*)pSubplan, NULL, pPlan); + if (TSDB_CODE_SUCCESS != code) { + break; + } } - SNode* pChild = NULL; - FOREACH(pChild, pLogicSubplan->pChildren) { - CHECK_CODE_EXT(doBuildPhysiPlan(pCxt, (SSubLogicPlan*)pChild, pSubplan, pQueryPlan)); + if (TSDB_CODE_SUCCESS == code) { + *pPhysiPlan = pPlan; + } else { + nodesDestroyNode(pPlan); } - return TSDB_CODE_SUCCESS; -} - -static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) { - *pPlan = makeQueryPhysiPlan(pCxt); - CHECK_ALLOC(*pPlan, TSDB_CODE_OUT_OF_MEMORY); - SNode* pSubplan = NULL; - FOREACH(pSubplan, pLogicPlan->pTopSubplans) { - CHECK_CODE_EXT(doBuildPhysiPlan(pCxt, (SSubLogicPlan*)pSubplan, NULL, *pPlan)); - } - return TSDB_CODE_SUCCESS; + return code; } -int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan, SArray* pExecNodeList) { +int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) { SPhysiPlanContext cxt = { .pPlanCxt = pCxt, .errCode = TSDB_CODE_SUCCESS, @@ -861,16 +956,5 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } - SQueryLogicPlan* pLogicPlan = NULL; - SSubLogicPlan* pSubLogicPlan = NULL; - int32_t code = splitLogicPlan(&cxt, pLogicNode, &pSubLogicPlan); - if (TSDB_CODE_SUCCESS == code) { - code = scaleOutLogicPlan(&cxt, pSubLogicPlan, &pLogicPlan); - } - if (TSDB_CODE_SUCCESS == code) { - code = buildPhysiPlan(&cxt, pLogicPlan, pPlan); - } - nodesDestroyNode(pSubLogicPlan); - nodesDestroyNode(pLogicPlan); - return code; + return doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); } diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c new file mode 100644 index 0000000000000000000000000000000000000000..ca6c7a2577e3439bcb50c907ea126ea6fb265b46 --- /dev/null +++ b/source/libs/planner/src/planScaleOut.c @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planInt.h" + +typedef struct SScaleOutContext { + SPlanContext* pPlanCxt; + int32_t subplanId; +} SScaleOutContext; + +static SLogicSubplan* singleCloneSubLogicPlan(SScaleOutContext* pCxt, SLogicSubplan* pSrc, int32_t level) { + SLogicSubplan* pDst = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pDst) { + return NULL; + } + pDst->pNode = nodesCloneNode(pSrc->pNode); + if (NULL == pDst->pNode) { + nodesDestroyNode(pDst); + return NULL; + } + pDst->subplanType = pSrc->subplanType; + pDst->level = level; + pDst->id.queryId = pSrc->id.queryId; + pDst->id.groupId = pSrc->id.groupId; + pDst->id.subplanId = pCxt->subplanId++; + return pDst; +} + +static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; + size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); + for (int32_t i = 0; i < numOfVgroups; ++i) { + SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + if (NULL == pNewSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, pNewSubplan)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); +} + +static int32_t doSetScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + pScan->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + if (NULL == pScan->pVgroupList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pScan->pVgroupList->vgroups, pVgroup, sizeof(SVgroupInfo)); + *pFound = true; + return TSDB_CODE_SUCCESS; + } + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + int32_t code = doSetScanVgroup((SLogicNode*)pChild, pVgroup, pFound); + if (TSDB_CODE_SUCCESS != code || *pFound) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup) { + bool found = false; + return doSetScanVgroup(pNode, pVgroup, &found); +} + +static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + if (pSubplan->pVgroupList && !pCxt->pPlanCxt->streamQuery) { + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { + SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + if (NULL == pNewSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + code = setScanVgroup(pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pGroup, pNewSubplan); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; + } else { + return scaleOutForMerge(pCxt, pSubplan, level, pGroup); + } +} + +static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { + int32_t code = TSDB_CODE_SUCCESS; + bool topLevel = (0 == LIST_LENGTH(pParentsGroup)); + SNode* pChild = NULL; + FOREACH(pChild, pCurrentGroup) { + if (topLevel) { + code = nodesListAppend(pParentsGroup, pChild); + } else { + SNode* pParent = NULL; + FOREACH(pParent, pParentsGroup) { + code = nodesListMakeAppend(&(((SLogicSubplan*)pParent)->pChildren), pChild); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&(((SLogicSubplan*)pChild)->pParents), pParent); + } + } + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t* pLevel, SNodeList* pParentsGroup) { + SNodeList* pCurrentGroup = nodesMakeList(); + if (NULL == pCurrentGroup) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + switch (pSubplan->subplanType) { + case SUBPLAN_TYPE_MERGE: + code = scaleOutForMerge(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + case SUBPLAN_TYPE_SCAN: + code = scaleOutForScan(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + case SUBPLAN_TYPE_MODIFY: + code = scaleOutForModify(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + default: + break; + } + + if (TSDB_CODE_SUCCESS == code) { + code = pushHierarchicalPlan(pParentsGroup, pCurrentGroup); + ++(*pLevel); + } + + if (TSDB_CODE_SUCCESS == code) { + SNode* pChild; + FOREACH(pChild, pSubplan->pChildren) { + code = doScaleOut(pCxt, (SLogicSubplan*)pChild, pLevel, pCurrentGroup); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + } + + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pCurrentGroup); + } + + return code; +} + +static SQueryLogicPlan* makeQueryLogicPlan() { + SQueryLogicPlan* pLogicPlan = (SQueryLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN); + if (NULL == pLogicPlan) { + return NULL; + } + pLogicPlan->pTopSubplans = nodesMakeList(); + if (NULL == pLogicPlan->pTopSubplans) { + nodesDestroyNode(pLogicPlan); + return NULL; + } + return pLogicPlan; +} + +int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan) { + SQueryLogicPlan* pPlan = makeQueryLogicPlan(); + if (NULL == pPlan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SScaleOutContext cxt = { .pPlanCxt = pCxt, .subplanId = 1 }; + int32_t code = doScaleOut(&cxt, pLogicSubplan, &(pPlan->totalLevel), pPlan->pTopSubplans); + if (TSDB_CODE_SUCCESS == code) { + *pLogicPlan = pPlan; + } else { + nodesDestroyNode(pPlan); + } + + return code; +} diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 1d0cbf22df0e245776ec7b1128a09fb0eb2364fd..b7a99d365d76c5e5f2b1ca949e557b2c145c249c 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -29,7 +29,7 @@ typedef struct SSplitContext { void* pInfo; } SSplitContext; -typedef int32_t (*FMatch)(SSplitContext* pCxt, SSubLogicPlan* pSubplan); +typedef int32_t (*FMatch)(SSplitContext* pCxt, SLogicSubplan* pSubplan); typedef int32_t (*FSplit)(SSplitContext* pCxt); typedef struct SSplitRule { @@ -40,11 +40,12 @@ typedef struct SSplitRule { typedef struct SStsInfo { SScanLogicNode* pScan; - SSubLogicPlan* pSubplan; + SLogicSubplan* pSubplan; } SStsInfo; static SLogicNode* stsMatchByNode(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && + NULL != ((SScanLogicNode*)pNode)->pVgroupList && ((SScanLogicNode*)pNode)->pVgroupList->numOfVgroups > 1) { return pNode; } SNode* pChild; @@ -57,13 +58,13 @@ static SLogicNode* stsMatchByNode(SLogicNode* pNode) { return NULL; } -static int32_t stsMatch(SSplitContext* pCxt, SSubLogicPlan* pSubplan) { +static int32_t stsMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan) { if (SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS)) { return TSDB_CODE_SUCCESS; } SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { - SStsInfo* pInfo = calloc(1, sizeof(SStsInfo)); + SStsInfo* pInfo = taosMemoryCalloc(1, sizeof(SStsInfo)); CHECK_ALLOC(pInfo, TSDB_CODE_OUT_OF_MEMORY); pInfo->pScan = (SScanLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; @@ -73,7 +74,7 @@ static int32_t stsMatch(SSplitContext* pCxt, SSubLogicPlan* pSubplan) { } SNode* pChild; FOREACH(pChild, pSubplan->pChildren) { - int32_t code = stsMatch(pCxt, (SSubLogicPlan*)pChild); + int32_t code = stsMatch(pCxt, (SLogicSubplan*)pChild); if (TSDB_CODE_SUCCESS != code || pCxt->match) { return code; } @@ -81,8 +82,8 @@ static int32_t stsMatch(SSplitContext* pCxt, SSubLogicPlan* pSubplan) { return TSDB_CODE_SUCCESS; } -static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan) { - SSubLogicPlan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); +static SLogicSubplan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan) { + SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { return NULL; } @@ -94,7 +95,7 @@ static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* return pSubplan; } -static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SSubLogicPlan* pSubplan, SScanLogicNode* pScan) { +static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan) { SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; @@ -105,6 +106,8 @@ static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SSubLogicPlan* pSubpla return TSDB_CODE_OUT_OF_MEMORY; } + pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + if (NULL == pScan->node.pParent) { pSubplan->pNode = (SLogicNode*)pExchange; return TSDB_CODE_SUCCESS; @@ -144,7 +147,7 @@ static const SSplitRule splitRuleSet[] = { static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); -int32_t applySplitRule(SSubLogicPlan* pSubplan) { +static int32_t applySplitRule(SLogicSubplan* pSubplan) { SSplitContext cxt = { .errCode = TSDB_CODE_SUCCESS, .groupId = pSubplan->id.groupId + 1, .match = false, .pInfo = NULL }; bool split = false; do { @@ -163,3 +166,45 @@ int32_t applySplitRule(SSubLogicPlan* pSubplan) { } while (split); return TSDB_CODE_SUCCESS; } + +static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { + pNode->pParent = pParent; + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + doSetLogicNodeParent((SLogicNode*)pChild, pNode); + } +} + +static void setLogicNodeParent(SLogicNode* pNode) { + doSetLogicNodeParent(pNode, NULL); +} + +int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) { + SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSubplan->pNode = nodesCloneNode(pLogicNode); + if (NULL == pSubplan->pNode) { + nodesDestroyNode(pSubplan); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) { + pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; + TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)pSubplan->pNode)->pDataBlocks, SArray*); + } else { + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + } + pSubplan->id.queryId = pCxt->queryId; + setLogicNodeParent(pSubplan->pNode); + + int32_t code = applySplitRule(pSubplan); + if (TSDB_CODE_SUCCESS == code) { + *pLogicSubplan = pSubplan; + } else { + nodesDestroyNode(pSubplan); + } + + return code; +} \ No newline at end of file diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index fa0dc549c8a0d3ea1dcaa4120a9c9e0d020cb3b5..bcea94278ea7c686e44045ea9470152e5052556c 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -17,20 +17,29 @@ #include "planInt.h" -int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode) { - return TSDB_CODE_SUCCESS; -} - int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) { SLogicNode* pLogicNode = NULL; + SLogicSubplan* pLogicSubplan = NULL; + SQueryLogicPlan* pLogicPlan = NULL; + int32_t code = createLogicPlan(pCxt, &pLogicNode); if (TSDB_CODE_SUCCESS == code) { - code = optimize(pCxt, pLogicNode); + code = optimizeLogicPlan(pCxt, pLogicNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = splitLogicPlan(pCxt, pLogicNode, &pLogicSubplan); + } + if (TSDB_CODE_SUCCESS == code) { + code = scaleOutLogicPlan(pCxt, pLogicSubplan, &pLogicPlan); } if (TSDB_CODE_SUCCESS == code) { - code = createPhysiPlan(pCxt, pLogicNode, pPlan, pExecNodeList); + code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList); } + nodesDestroyNode(pLogicNode); + nodesDestroyNode(pLogicSubplan); + nodesDestroyNode(pLogicPlan); + terrno = code; return code; } diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 8baa64d8fb78b92bbee314442ccf2f0b30cc2213..b4c0e43a2d97693a9b42087f985fbb17873f52b5 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -17,6 +17,7 @@ #include +#include "cmdnodes.h" #include "parser.h" #include "planInt.h" @@ -25,11 +26,6 @@ using namespace testing; class PlannerTest : public Test { protected: - enum TestTarget { - TEST_LOGIC_PLAN, - TEST_PHYSICAL_PLAN - }; - void setDatabase(const string& acctId, const string& db) { acctId_ = acctId; db_ = db; @@ -45,7 +41,7 @@ protected: cxt_.pSql = sqlBuf_.c_str(); } - bool run(TestTarget target = TEST_PHYSICAL_PLAN) { + bool run(bool streamQuery = false) { int32_t code = qParseQuerySql(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { @@ -55,36 +51,54 @@ protected: const string syntaxTreeStr = toString(query_->pRoot, false); - SLogicNode* pLogicPlan = nullptr; - SPlanContext cxt = { .queryId = 1, .acctId = 0, .pAstRoot = query_->pRoot }; - code = createLogicPlan(&cxt, &pLogicPlan); + SLogicNode* pLogicNode = nullptr; + SPlanContext cxt = {0}; + cxt.queryId = 1; + cxt.acctId = 0; + cxt.streamQuery = streamQuery; + + setPlanContext(query_, &cxt); + code = createLogicPlan(&cxt, &pLogicNode); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl; + cout << "sql:[" << cxt_.pSql << "] createLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; return false; } cout << "====================sql : [" << cxt_.pSql << "]" << endl; - cout << "syntax test : " << endl; + cout << "syntax tree : " << endl; cout << syntaxTreeStr << endl; cout << "unformatted logic plan : " << endl; - cout << toString((const SNode*)pLogicPlan, false) << endl; - - if (TEST_PHYSICAL_PLAN == target) { - SQueryPlan* pPlan = nullptr; - code = createPhysiPlan(&cxt, pLogicPlan, &pPlan, NULL); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] physical plan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - cout << "unformatted physical plan : " << endl; - cout << toString((const SNode*)pPlan, false) << endl; - SNode* pNode; - FOREACH(pNode, pPlan->pSubplans) { - SNode* pSubplan; - FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { - cout << "unformatted physical subplan : " << endl; - cout << toString(pSubplan, false) << endl; - } + cout << toString((const SNode*)pLogicNode, false) << endl; + + SLogicSubplan* pLogicSubplan = nullptr; + code = splitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] splitLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + + SQueryLogicPlan* pLogicPlan = NULL; + code = scaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] createPhysiPlan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + + SQueryPlan* pPlan = nullptr; + code = createPhysiPlan(&cxt, pLogicPlan, &pPlan, NULL); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] createPhysiPlan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + + cout << "unformatted physical plan : " << endl; + cout << toString((const SNode*)pPlan, false) << endl; + SNode* pNode; + FOREACH(pNode, pPlan->pSubplans) { + SNode* pSubplan; + FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { + cout << "unformatted physical subplan : " << endl; + cout << toString(pSubplan, false) << endl; } } @@ -94,6 +108,20 @@ protected: private: static const int max_err_len = 1024; + void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { + if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { + pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; + pCxt->topicQuery = true; + } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { + SMCreateSmaReq req = {0}; + tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); + nodesStringToNode(req.ast, &pCxt->pAstRoot); + pCxt->streamQuery = true; + } else { + pCxt->pAstRoot = pQuery->pRoot; + } + } + void reset() { memset(&cxt_, 0, sizeof(cxt_)); memset(errMagBuf_, 0, max_err_len); @@ -109,16 +137,8 @@ private: cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl; return string(); } - SNode* pNode; - code = nodesStringToNode(pStr, &pNode); - if (code != TSDB_CODE_SUCCESS) { - tfree(pStr); - cout << "sql:[" << cxt_.pSql << "] toObject code:" << code << ", strerror:" << tstrerror(code) << endl; - return string(); - } - nodesDestroyNode(pNode); string str(pStr); - tfree(pStr); + taosMemoryFreeClear(pStr); return str; } @@ -173,3 +193,37 @@ TEST_F(PlannerTest, interval) { bind("SELECT count(*) FROM t1 interval(10s)"); ASSERT_TRUE(run()); } + +TEST_F(PlannerTest, sessionWindow) { + setDatabase("root", "test"); + + bind("SELECT count(*) FROM t1 session(ts, 10s)"); + ASSERT_TRUE(run()); +} + +TEST_F(PlannerTest, showTables) { + setDatabase("root", "test"); + + bind("show tables"); +} + +TEST_F(PlannerTest, createTopic) { + setDatabase("root", "test"); + + bind("create topic tp as SELECT * FROM st1"); + ASSERT_TRUE(run()); +} + +TEST_F(PlannerTest, stream) { + setDatabase("root", "test"); + + bind("SELECT sum(c1) FROM st1"); + ASSERT_TRUE(run(true)); +} + +TEST_F(PlannerTest, createSmaIndex) { + setDatabase("root", "test"); + + bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); + ASSERT_TRUE(run()); +} diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 63fbf59c064a91a883a0ae52ad7fb9e1c56b1c55..0cf46edf11aefd0f114ac1d302968b016256fe18 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -140,7 +140,7 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) return 0; } -int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { +int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *rpcCtx) { char* pMsg = rpcMallocCont(pInfo->msgInfo.len); if (NULL == pMsg) { qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); @@ -154,10 +154,59 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp .contLen = pInfo->msgInfo.len, .ahandle = (void*)pInfo, .handle = pInfo->msgInfo.handle, + .persistHandle = persistHandle, .code = 0}; + if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH || + pInfo->msgType == TDMT_VND_QUERY_CONTINUE) { + rpcMsg.persistHandle = 1; + } assert(pInfo->fp != NULL); - rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId); + rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); return TSDB_CODE_SUCCESS; } + +int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { + return asyncSendMsgToServerExt(pTransporter, epSet, pTransporterId, pInfo, false, NULL); +} + +char *jobTaskStatusStr(int32_t status) { + switch (status) { + case JOB_TASK_STATUS_NULL: + return "NULL"; + case JOB_TASK_STATUS_NOT_START: + return "NOT_START"; + case JOB_TASK_STATUS_EXECUTING: + return "EXECUTING"; + case JOB_TASK_STATUS_PARTIAL_SUCCEED: + return "PARTIAL_SUCCEED"; + case JOB_TASK_STATUS_SUCCEED: + return "SUCCEED"; + case JOB_TASK_STATUS_FAILED: + return "FAILED"; + case JOB_TASK_STATUS_CANCELLING: + return "CANCELLING"; + case JOB_TASK_STATUS_CANCELLED: + return "CANCELLED"; + case JOB_TASK_STATUS_DROPPING: + return "DROPPING"; + default: + break; + } + + return "UNKNOWN"; +} + +SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name) { + SSchema s = {0}; + s.type = type; + s.bytes = bytes; + s.colId = colId; + + tstrncpy(s.name, name, tListLen(s.name)); + return s; +} + + + diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 37e8b7302eb16de21a0813250ad68a559d1e8420..20eb49ed330cc3d742c27086085a4de728b688b2 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -27,13 +27,19 @@ int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { memcpy(pOut->db, usedbRsp->db, TSDB_DB_FNAME_LEN); pOut->dbId = usedbRsp->uid; - pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); + + pOut->dbVgroup = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; + + if (usedbRsp->vgNum <= 0) { + return TSDB_CODE_SUCCESS; + } + pOut->dbVgroup->vgHash = taosHashInit(usedbRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pOut->dbVgroup->vgHash) { @@ -145,7 +151,7 @@ PROCESS_USEDB_OVER: if (code != 0) { if (pOut) { if (pOut->dbVgroup) taosHashCleanup(pOut->dbVgroup->vgHash); - tfree(pOut->dbVgroup); + taosMemoryFreeClear(pOut->dbVgroup); } qError("failed to process usedb rsp since %s", terrstr()); } @@ -166,7 +172,7 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) { } if (pMetaMsg->tableType != TSDB_SUPER_TABLE && pMetaMsg->tableType != TSDB_CHILD_TABLE && - pMetaMsg->tableType != TSDB_NORMAL_TABLE) { + pMetaMsg->tableType != TSDB_NORMAL_TABLE && pMetaMsg->tableType != TSDB_SYSTEM_TABLE) { qError("invalid tableType[%d] in table meta rsp msg", pMetaMsg->tableType); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -193,7 +199,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STabl int32_t total = msg->numOfColumns + msg->numOfTags; int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; - STableMeta *pTableMeta = calloc(1, metaSize); + STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); return TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp index 72ce0f7c37f814e0457699ad2c1e7b1d883bab6b..9615557c883f6adf4a22f6a1d14417e4c5fe9839 100644 --- a/source/libs/qcom/test/queryTest.cpp +++ b/source/libs/qcom/test/queryTest.cpp @@ -32,13 +32,13 @@ typedef struct SParam { int32_t testPrint(void* p) { SParam* param = (SParam*)p; printf("hello world, %d\n", param->v); - tfree(p); + taosMemoryFreeClear(p); return 0; } int32_t testPrintError(void* p) { SParam* param = (SParam*) p; - tfree(p); + taosMemoryFreeClear(p); return -1; } @@ -61,14 +61,14 @@ int main(int argc, char** argv) { } TEST(testCase, async_task_test) { - SParam* p = (SParam*)calloc(1, sizeof(SParam)); + SParam* p = (SParam*)taosMemoryCalloc(1, sizeof(SParam)); taosAsyncExec(testPrint, p, NULL); taosMsleep(5); } TEST(testCase, many_async_task_test) { for(int32_t i = 0; i < 50; ++i) { - SParam* p = (SParam*) calloc(1, sizeof(SParam)); + SParam* p = (SParam*) taosMemoryCalloc(1, sizeof(SParam)); p->v = i; taosAsyncExec(testPrint, p, NULL); } @@ -78,7 +78,7 @@ TEST(testCase, many_async_task_test) { TEST(testCase, error_in_async_test) { int32_t code = 0; - SParam* p = (SParam*) calloc(1, sizeof(SParam)); + SParam* p = (SParam*) taosMemoryCalloc(1, sizeof(SParam)); taosAsyncExec(testPrintError, p, &code); taosMsleep(1); printf("Error code:%d after asynchronously exec function\n", code); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index b5b8726a4c9115541c666ad0c66531700ae885d5..573eaed2e6bec1065d57425493d72c235966ab9a 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "qworker.h" #include "tlockfree.h" #include "ttimer.h" @@ -59,46 +60,40 @@ enum { QW_WRITE, }; -enum { - QW_EXIST_ACQUIRE = 1, - QW_EXIST_RET_ERR, -}; enum { QW_NOT_EXIST_RET_ERR = 1, QW_NOT_EXIST_ADD, }; -enum { - QW_ADD_RET_ERR = 1, - QW_ADD_ACQUIRE, -}; - typedef struct SQWDebug { - int32_t lockDebug; + bool lockEnable; + bool statusEnable; + bool dumpEnable; } SQWDebug; +typedef struct SQWConnInfo { + void *handle; + void *ahandle; +} SQWConnInfo; + typedef struct SQWMsg { - void *node; - char *msg; - int32_t msgLen; - void *connection; + void *node; + char *msg; + int32_t msgLen; + SQWConnInfo connInfo; } SQWMsg; typedef struct SQWHbInfo { SSchedulerHbRsp rsp; - void *connection; + SQWConnInfo connInfo; } SQWHbInfo; typedef struct SQWPhaseInput { - int8_t taskStatus; - int8_t taskType; int32_t code; } SQWPhaseInput; typedef struct SQWPhaseOutput { - int32_t rspCode; - bool needStop; } SQWPhaseOutput; @@ -112,17 +107,15 @@ typedef struct SQWTaskCtx { SRWLatch lock; int8_t phase; int8_t taskType; - - void *readyConnection; - void *dropConnection; - void *cancelConnection; bool emptyRes; - bool multiExec; - int8_t queryContinue; - int8_t queryInQueue; + bool queryFetched; + bool queryEnd; + bool queryContinue; + bool queryInQueue; int32_t rspCode; + SQWConnInfo connInfo; int8_t events[QW_EVENT_MAX]; qTaskInfo_t taskHandle; @@ -130,27 +123,26 @@ typedef struct SQWTaskCtx { } SQWTaskCtx; typedef struct SQWSchStatus { - int32_t lastAccessTs; // timestamp in second - uint64_t hbSeqId; - void *hbConnection; - SRWLatch tasksLock; - SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus + int32_t lastAccessTs; // timestamp in second + SRWLatch hbConnLock; + SQWConnInfo hbConnInfo; + SQueryNodeEpId hbEpId; + SRWLatch tasksLock; + SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus } SQWSchStatus; // Qnode/Vnode level task management typedef struct SQWorkerMgmt { - SQWorkerCfg cfg; - int8_t nodeType; - int32_t nodeId; - void *timer; - tmr_h hbTimer; - SRWLatch schLock; - //SRWLatch ctxLock; - SHashObj *schHash; //key: schedulerId, value: SQWSchStatus - SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx - void *nodeObj; - putReqToQueryQFp putToQueueFp; - sendReqToDnodeFp sendReqFp; + SQWorkerCfg cfg; + int8_t nodeType; + int32_t nodeId; + void *timer; + tmr_h hbTimer; + SRWLatch schLock; + // SRWLatch ctxLock; + SHashObj *schHash; // key: schedulerId, value: SQWSchStatus + SHashObj *ctxHash; // key: queryId+taskId, value: SQWTaskCtx + SMsgCb msgCb; } SQWorkerMgmt; #define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId @@ -184,12 +176,16 @@ typedef struct SQWorkerMgmt { #define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__) #define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__) +#define QW_DUMP(param, ...) do { if (gQWDebug.dumpEnable) { qDebug("QW:%p " param, mgmt, __VA_ARGS__); } } while (0) + + #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) #define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOGL(param, ...) qDebugL("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) #define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) @@ -199,7 +195,7 @@ typedef struct SQWorkerMgmt { #define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) +#define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockEnable) { qDebug(__VA_ARGS__); } } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 @@ -235,8 +231,6 @@ typedef struct SQWorkerMgmt { } \ } while (0) -int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); - #ifdef __cplusplus } #endif diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index ecb5dbd654d24292f818ee221105a1e9d79dd798..be1d47a189b9233f29b155deb8c8cf9f59cfae98 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -30,17 +30,18 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req); -int32_t qwBuildAndSendDropRsp(void *connection, int32_t code); -int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); -int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); +int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code); +int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code); +int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete); -int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection); -int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code); -int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code); +int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn); +int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code); +int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code); void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); -int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *rsp, int32_t code); +int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); +int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 42890ab38a09753ae1607d2cc58cfdf9ec080f1f..70ca0f736bdf04d832aad7ba944d599748ebe253 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -9,12 +9,21 @@ #include "tname.h" #include "dataSinkMgt.h" -SQWDebug gQWDebug = {0}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; + +int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { + if (!gQWDebug.statusEnable) { + return TSDB_CODE_SUCCESS; + } -int32_t qwValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus) { int32_t code = 0; if (oriStatus == newStatus) { + if (newStatus == JOB_TASK_STATUS_EXECUTING || newStatus == JOB_TASK_STATUS_FAILED) { + *ignore = true; + return TSDB_CODE_SUCCESS; + } + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -47,19 +56,27 @@ int32_t qwValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus) { case JOB_TASK_STATUS_PARTIAL_SUCCEED: if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_SUCCEED - && newStatus != JOB_TASK_STATUS_CANCELLED) { + && newStatus != JOB_TASK_STATUS_CANCELLED + && newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_DROPPING) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; case JOB_TASK_STATUS_SUCCEED: if (newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_DROPPING) { + && newStatus != JOB_TASK_STATUS_DROPPING + && newStatus != JOB_TASK_STATUS_FAILED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; case JOB_TASK_STATUS_FAILED: + if (newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_DROPPING) { + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + break; + case JOB_TASK_STATUS_CANCELLING: if (newStatus != JOB_TASK_STATUS_CANCELLED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); @@ -68,11 +85,13 @@ int32_t qwValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus) { break; case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_DROPPING: - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + if (newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } break; default: - QW_TASK_ELOG("invalid task status:%d", oriStatus); + QW_TASK_ELOG("invalid task origStatus:%s", jobTaskStatusStr(oriStatus)); return TSDB_CODE_QRY_APP_ERROR; } @@ -80,24 +99,95 @@ int32_t qwValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus) { _return: - QW_TASK_ELOG("invalid task status update from %d to %d", oriStatus, newStatus); + QW_TASK_ELOG("invalid task status update from %s to %s", jobTaskStatusStr(oriStatus), jobTaskStatusStr(newStatus)); QW_RET(code); } +void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) { + +} + +void qwDbgDumpMgmtInfo(SQWorkerMgmt *mgmt) { + if (!gQWDebug.dumpEnable) { + return; + } + + QW_LOCK(QW_READ, &mgmt->schLock); + + QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash)); + + void *key = NULL; + size_t keyLen = 0; + int32_t i = 0; + SQWSchStatus *sch = NULL; + + void *pIter = taosHashIterate(mgmt->schHash, NULL); + while (pIter) { + sch = (SQWSchStatus *)pIter; + qwDbgDumpSchInfo(sch, i); + ++i; + pIter = taosHashIterate(mgmt->schHash, pIter); + } + + QW_UNLOCK(QW_READ, &mgmt->schLock); + + QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash)); +} + +char *qwPhaseStr(int32_t phase) { + switch (phase) { + case QW_PHASE_PRE_QUERY: + return "PRE_QUERY"; + case QW_PHASE_POST_QUERY: + return "POST_QUERY"; + case QW_PHASE_PRE_FETCH: + return "PRE_FETCH"; + case QW_PHASE_POST_FETCH: + return "POST_FETCH"; + case QW_PHASE_PRE_CQUERY: + return "PRE_CQUERY"; + case QW_PHASE_POST_CQUERY: + return "POST_CQUERY"; + default: + break; + } + + return "UNKNOWN"; +} + +char *qwBufStatusStr(int32_t bufStatus) { + switch (bufStatus) { + case DS_BUF_LOW: + return "LOW"; + case DS_BUF_FULL: + return "FULL"; + case DS_BUF_EMPTY: + return "EMPTY"; + default: + break; + } + + return "UNKNOWN"; +} + int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) { int32_t code = 0; int8_t origStatus = 0; + bool ignore = false; while (true) { origStatus = atomic_load_8(&task->status); - QW_ERR_RET(qwValidateStatus(QW_FPARAMS(), origStatus, status)); + QW_ERR_RET(qwDbgValidateStatus(QW_FPARAMS(), origStatus, status, &ignore)); + if (ignore) { + break; + } if (origStatus != atomic_val_compare_exchange_8(&task->status, origStatus, status)) { continue; } - QW_TASK_DLOG("task status updated from %d to %d", origStatus, status); + QW_TASK_DLOG("task status updated from %s to %s", jobTaskStatusStr(origStatus), jobTaskStatusStr(status)); break; } @@ -106,7 +196,7 @@ int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) { } -int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { @@ -140,7 +230,7 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, QW_UNLOCK(rwType, &mgmt->schLock); if (QW_NOT_EXIST_ADD == nOpt) { - QW_ERR_RET(qwAddSchedulerImpl(mgmt, sId, rwType, sch)); + QW_ERR_RET(qwAddSchedulerImpl(mgmt, sId, rwType)); nOpt = QW_NOT_EXIST_RET_ERR; @@ -206,16 +296,18 @@ int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, i if (rwType && task) { QW_RET(qwAcquireTaskStatus(QW_FPARAMS(), rwType, sch, task)); } else { - QW_TASK_ELOG("task status already exist, id:%s", id); + QW_TASK_ELOG("task status already exist, newStatus:%s", jobTaskStatusStr(status)); QW_ERR_RET(TSDB_CODE_QRY_TASK_ALREADY_EXIST); } } else { - QW_TASK_ELOG("taosHashPut to tasksHash failed, code:%x", code); + QW_TASK_ELOG("taosHashPut to tasksHash failed, error:%x - %s", code, tstrerror(code)); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } QW_UNLOCK(QW_WRITE, &sch->tasksLock); + QW_TASK_DLOG("task status added, newStatus:%s", jobTaskStatusStr(status)); + if (rwType && task) { QW_ERR_RET(qwAcquireTaskStatus(QW_FPARAMS(), rwType, sch, task)); } @@ -251,11 +343,9 @@ void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - - //QW_LOCK(rwType, &mgmt->ctxLock); + *ctx = taosHashAcquire(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { - //QW_UNLOCK(rwType, &mgmt->ctxLock); QW_TASK_DLOG_E("task ctx not exist, may be dropped"); QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } @@ -276,32 +366,28 @@ int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, int32_t status, SQWTaskCtx **ctx) { +int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); SQWTaskCtx nctx = {0}; - //QW_LOCK(QW_WRITE, &mgmt->ctxLock); int32_t code = taosHashPut(mgmt->ctxHash, id, sizeof(id), &nctx, sizeof(SQWTaskCtx)); if (0 != code) { - //QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); - if (HASH_NODE_EXIST(code)) { if (acquire && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), ctx)); } else if (ctx) { QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } else { - QW_TASK_ELOG("task ctx already exist, id:%s", id); + QW_TASK_ELOG_E("task ctx already exist"); QW_ERR_RET(TSDB_CODE_QRY_TASK_ALREADY_EXIST); } } else { - QW_TASK_ELOG("taosHashPut to ctxHash failed, code:%x", code); + QW_TASK_ELOG("taosHashPut to ctxHash failed, error:%x", code); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } - //QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); if (acquire && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), ctx)); @@ -313,27 +399,19 @@ int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, int32_t status, SQWTaskCt } int32_t qwAddTaskCtx(QW_FPARAMS_DEF) { - QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), false, 0, NULL)); + QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), false, NULL)); } - - int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { - return qwAddTaskCtxImpl(QW_FPARAMS(), true, 0, ctx); + return qwAddTaskCtxImpl(QW_FPARAMS(), true, ctx); } -int32_t qwAddGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { - return qwAddTaskCtxImpl(QW_FPARAMS(), false, 0, ctx); -} - - void qwReleaseTaskCtx(SQWorkerMgmt *mgmt, void *ctx) { - //QW_UNLOCK(rwType, &mgmt->ctxLock); taosHashRelease(mgmt->ctxHash, ctx); } void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { - // RC WARNING + // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { qDestroyTask(otaskHandle); @@ -342,7 +420,7 @@ void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t code = 0; - // RC WARNING + // Note: free/kill may in RC qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); if (taskHandle && atomic_val_compare_exchange_ptr(&ctx->taskHandle, taskHandle, NULL)) { code = qAsyncKillTask(taskHandle); @@ -354,6 +432,9 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + rpcReleaseHandle(ctx->connInfo.handle, TAOS_CONN_SERVER); + ctx->connInfo.handle = NULL; + qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle); if (ctx->sinkHandle) { @@ -363,8 +444,7 @@ void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { } -// Note: NEED CTX HASH LOCKED BEFORE ENTRANCE -int32_t qwDropTaskCtx(QW_FPARAMS_DEF, int32_t rwType) { +int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); SQWTaskCtx octx; @@ -381,29 +461,18 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF, int32_t rwType) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_DROP); - if (rwType) { - QW_UNLOCK(rwType, &ctx->lock); - } - if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } - if (octx.taskHandle) { - qDestroyTask(octx.taskHandle); - } - - if (octx.sinkHandle) { - dsDestroyDataSinker(octx.sinkHandle); - } + qwFreeTask(QW_FPARAMS(), &octx); QW_TASK_DLOG_E("task ctx dropped"); return TSDB_CODE_SUCCESS; } - int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; @@ -433,7 +502,9 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { _return: - qwReleaseTaskStatus(QW_WRITE, sch); + if (task) { + qwReleaseTaskStatus(QW_WRITE, sch); + } qwReleaseScheduler(QW_WRITE, mgmt); QW_RET(code); @@ -451,12 +522,21 @@ int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) { _return: - qwReleaseTaskStatus(QW_READ, sch); + if (task) { + qwReleaseTaskStatus(QW_READ, sch); + } qwReleaseScheduler(QW_READ, mgmt); QW_RET(code); } +int32_t qwDropTask(QW_FPARAMS_DEF) { + QW_ERR_RET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_RET(qwDropTaskCtx(QW_FPARAMS())); + + return TSDB_CODE_SUCCESS; +} + int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { int32_t code = 0; bool qcontinue = true; @@ -469,17 +549,18 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { while (true) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); - + code = qExecTask(*taskHandle, &pRes, &useconds); if (code) { - QW_TASK_ELOG("qExecTask failed, code:%s", tstrerror(code)); - QW_ERR_JRET(code); + QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); } ++execNum; if (NULL == pRes) { - QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); + QW_TASK_DLOG("qExecTask end with empty res, useconds:%"PRIu64, useconds); + dsEndPut(sinkHandle, useconds); if (TASK_TYPE_TEMP == ctx->taskType) { @@ -493,16 +574,18 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { break; } + int32_t rows = pRes->info.rows; + ASSERT(pRes->info.rows > 0); SInputData inputData = {.pData = pRes}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); if (code) { - QW_TASK_ELOG("dsPutDataBlock failed, code:%s", tstrerror(code)); - QW_ERR_JRET(code); + QW_TASK_ELOG("dsPutDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); } - QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); + QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", rows, qcontinue); if (!qcontinue) { break; @@ -515,9 +598,11 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { break; } - } -_return: + if (atomic_load_32(&ctx->rspCode)) { + break; + } + } QW_RET(code); } @@ -525,6 +610,9 @@ _return: int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; + hbInfo->connInfo = sch->hbConnInfo; + hbInfo->rsp.epId = sch->hbEpId; + QW_LOCK(QW_READ, &sch->tasksLock); taskNum = taosHashGetSize(sch->tasksHash); @@ -536,9 +624,6 @@ int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbI return TSDB_CODE_QRY_OUT_OF_MEMORY; } - hbInfo->connection = sch->hbConnection; - hbInfo->rsp.seqId = -1; - void *key = NULL; size_t keyLen = 0; int32_t i = 0; @@ -574,10 +659,9 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void int32_t code = 0; if (ctx->emptyRes) { - QW_TASK_DLOG("query empty result, query end, phase:%d", ctx->phase); - - QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); + QW_TASK_DLOG_E("query end with empty result"); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED); QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); *rspMsg = rsp; @@ -598,34 +682,29 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void if (queryEnd) { code = dsGetDataBlock(ctx->sinkHandle, pOutput); if (code) { - QW_TASK_ELOG("dsGetDataBlock failed, code:%x", code); + QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); QW_ERR_RET(code); } - QW_TASK_DLOG("no data in sink and query end, phase:%d", ctx->phase); - - QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); + QW_TASK_DLOG_E("no data in sink and query end"); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED); QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); + *rspMsg = rsp; *dataLen = 0; - return TSDB_CODE_SUCCESS; } pOutput->bufStatus = DS_BUF_EMPTY; - - QW_TASK_DLOG("no res data in sink, need response later, queryEnd:%d", queryEnd); return TSDB_CODE_SUCCESS; } - // Got data from sink + QW_TASK_DLOG("there are data in sink, dataLength:%d", len); *dataLen = len; - - QW_TASK_DLOG("task got data in sink, dataLength:%d", len); QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); *rspMsg = rsp; @@ -633,13 +712,13 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void pOutput->pData = rsp->data; code = dsGetDataBlock(ctx->sinkHandle, pOutput); if (code) { - QW_TASK_ELOG("dsGetDataBlock failed, code:%x", code); + QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); QW_ERR_RET(code); } if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { - QW_SCH_TASK_DLOG("task all fetched, status:%d", JOB_TASK_STATUS_SUCCEED); - QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); + QW_TASK_DLOG_E("task all data fetched, done"); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED); } return TSDB_CODE_SUCCESS; @@ -647,15 +726,11 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; - int8_t status = 0; SQWTaskCtx *ctx = NULL; - bool locked = false; - void *dropConnection = NULL; - void *cancelConnection = NULL; - - QW_SCH_TASK_DLOG("start to handle event at phase %d", phase); + SQWConnInfo *dropConnection = NULL; + SQWConnInfo *cancelConnection = NULL; - output->needStop = false; + QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); if (QW_PHASE_PRE_QUERY == phase) { QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); @@ -664,193 +739,107 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } QW_LOCK(QW_WRITE, &ctx->lock); - locked = true; + + if (QW_PHASE_PRE_FETCH == phase) { + atomic_store_8((int8_t*)&ctx->queryFetched, true); + } else { + atomic_store_8(&ctx->phase, phase); + } + + if (atomic_load_8((int8_t*)&ctx->queryEnd)) { + QW_TASK_ELOG_E("query already end"); + QW_ERR_JRET(TSDB_CODE_QW_MSG_ERROR); + } switch (phase) { case QW_PHASE_PRE_QUERY: { - atomic_store_8(&ctx->phase, phase); - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { - QW_TASK_ELOG("task already cancelled/dropped at wrong phase, phase:%d", phase); - - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_ELOG("task already dropped at wrong phase %s", qwPhaseStr(phase)); + QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); break; } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); - - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - QW_SET_RSP_CODE(ctx, output->rspCode); - dropConnection = ctx->dropConnection; + dropConnection = &ctx->connInfo; + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + dropConnection = NULL; - // Note: ctx freed, no need to unlock it - locked = false; - - break; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_SET_RSP_CODE(ctx, output->rspCode); - - cancelConnection = ctx->cancelConnection; + qwBuildAndSendDropRsp(&ctx->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); break; } - if (ctx->rspCode) { - QW_TASK_ELOG("task already failed at wrong phase, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } - - if (!output->needStop) { - QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); - } + QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); break; } case QW_PHASE_PRE_FETCH: { - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task already dropped, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task dropping or already dropped, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (ctx->rspCode) { - QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - QW_TASK_WLOG("last fetch not finished, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_DUPLICATTED_OPERATION; + QW_TASK_WLOG("last fetch still not processed, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); } if (!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_READY)) { - QW_TASK_ELOG("query rsp are not ready, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_MSG_ERROR; + QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); } break; } case QW_PHASE_PRE_CQUERY: { - atomic_store_8(&ctx->phase, phase); - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task already dropped, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); - - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - output->needStop = true; - QW_SET_RSP_CODE(ctx, output->rspCode); - dropConnection = ctx->dropConnection; - - // Note: ctx freed, no need to unlock it - locked = false; - - QW_ERR_JRET(output->rspCode); - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - qwFreeTask(QW_FPARAMS(), ctx); + dropConnection = &ctx->connInfo; + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + dropConnection = NULL; + + qwBuildAndSendDropRsp(&ctx->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_SET_RSP_CODE(ctx, output->rspCode); - cancelConnection = ctx->cancelConnection; - - QW_ERR_JRET(output->rspCode); + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - - if (ctx->rspCode) { - QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } break; - } + } + default: + QW_TASK_ELOG("invalid phase %s", qwPhaseStr(phase)); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed at phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode, tstrerror(ctx->rspCode)); + QW_ERR_JRET(ctx->rspCode); } _return: if (ctx) { - if (output->rspCode) { - QW_UPDATE_RSP_CODE(ctx, output->rspCode); - } + QW_UPDATE_RSP_CODE(ctx, code); - if (locked) { - QW_UNLOCK(QW_WRITE, &ctx->lock); - } - + QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); } - if (code) { - output->needStop = true; - if (TSDB_CODE_SUCCESS == output->rspCode) { - output->rspCode = code; - } - } - if (dropConnection) { - qwBuildAndSendDropRsp(dropConnection, output->rspCode); - QW_TASK_DLOG("drop msg rsped, code:%x", output->rspCode); + qwBuildAndSendDropRsp(dropConnection, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", dropConnection->handle, code, tstrerror(code)); } if (cancelConnection) { - qwBuildAndSendCancelRsp(cancelConnection, output->rspCode); - QW_TASK_DLOG("cancel msg rsped, code:%x", output->rspCode); + qwBuildAndSendCancelRsp(cancelConnection, code); + QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code)); } - QW_SCH_TASK_DLOG("end to handle event at phase %d", phase); + QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code)); QW_RET(code); } @@ -858,170 +847,119 @@ _return: int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; - int8_t status = 0; SQWTaskCtx *ctx = NULL; - bool locked = false; - void *readyConnection = NULL; - void *dropConnection = NULL; - void *cancelConnection = NULL; - - QW_SCH_TASK_DLOG("start to handle event at phase %d", phase); + SQWConnInfo connInfo = {0}; + SQWConnInfo *readyConnection = NULL; - output->needStop = false; + QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); - locked = true; if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task already dropped, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (input->code) { - output->rspCode = input->code; - } if (QW_PHASE_POST_QUERY == phase) { if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { ctx->emptyRes = true; } - + +#if 0 if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { - readyConnection = ctx->readyConnection; + readyConnection = &ctx->connInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); } +#else + connInfo.handle = ctx->connInfo.handle; + readyConnection = &connInfo; + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); +#endif } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); - - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - output->needStop = true; - QW_SET_RSP_CODE(ctx, output->rspCode); - dropConnection = ctx->dropConnection; - - // Note: ctx freed, no need to unlock it - locked = false; + if (QW_PHASE_POST_FETCH == phase) { + QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } - QW_ERR_JRET(output->rspCode); - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - qwFreeTask(QW_FPARAMS(), ctx); + qwBuildAndSendDropRsp(&ctx->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_SET_RSP_CODE(ctx, output->rspCode); - cancelConnection = ctx->cancelConnection; - - QW_ERR_JRET(output->rspCode); + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } if (ctx->rspCode) { - QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); + QW_TASK_ELOG("task already failed, phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode, tstrerror(ctx->rspCode)); + QW_ERR_JRET(ctx->rspCode); } - if (QW_PHASE_POST_QUERY == phase && (!output->needStop)) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), input->taskStatus)); - } + QW_ERR_JRET(input->code); _return: + if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) { + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PARTIAL_SUCCEED); + } + if (ctx) { - if (output->rspCode) { - QW_UPDATE_RSP_CODE(ctx, output->rspCode); - } + QW_UPDATE_RSP_CODE(ctx, code); if (QW_PHASE_POST_FETCH != phase) { atomic_store_8(&ctx->phase, phase); } - if (locked) { - QW_UNLOCK(QW_WRITE, &ctx->lock); - } - + QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); } - if (code) { - output->needStop = true; - if (TSDB_CODE_SUCCESS == output->rspCode) { - output->rspCode = code; - } - } - if (readyConnection) { - qwBuildAndSendReadyRsp(readyConnection, output->rspCode); - QW_TASK_DLOG("ready msg rsped, code:%x", output->rspCode); - } - - if (dropConnection) { - qwBuildAndSendDropRsp(dropConnection, output->rspCode); - QW_TASK_DLOG("drop msg rsped, code:%x", output->rspCode); + qwBuildAndSendReadyRsp(readyConnection, code); + QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", readyConnection->handle, code, tstrerror(code)); } - if (cancelConnection) { - qwBuildAndSendCancelRsp(cancelConnection, output->rspCode); - QW_TASK_DLOG("cancel msg rsped, code:%x", output->rspCode); + if (code) { + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAILED); } - QW_SCH_TASK_DLOG("end to handle event at phase %d", phase); + QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code)); QW_RET(code); } - int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { int32_t code = 0; bool queryRsped = false; - bool needStop = false; struct SSubplan *plan = NULL; - int32_t rspCode = 0; SQWPhaseInput input = {0}; - SQWPhaseOutput output = {0}; qTaskInfo_t pTaskInfo = NULL; DataSinkHandle sinkHandle = NULL; SQWTaskCtx *ctx = NULL; - QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, &output)); + QW_ERR_JRET(qwRegisterBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); - needStop = output.needStop; - code = output.rspCode; - - if (needStop) { - QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_QUERY); - QW_ERR_JRET(code); - } + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); atomic_store_8(&ctx->taskType, taskType); - + atomic_store_ptr(&ctx->connInfo.handle, qwMsg->connInfo.handle); + atomic_store_ptr(&ctx->connInfo.ahandle, qwMsg->connInfo.ahandle); + + QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("task string to subplan failed, code:%s", tstrerror(code)); + QW_TASK_ELOG("task string to subplan failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); } - code = qCreateExecTask(qwMsg->node, 0, tId, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle); + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle); if (code) { - QW_TASK_ELOG("qCreateExecTask failed, code:%s", tstrerror(code)); + QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); } @@ -1030,10 +968,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - //TODO OPTIMIZE EMTYP RESULT QUERY RSP TO AVOID FURTHER FETCH - - QW_ERR_JRET(qwBuildAndSendQueryRsp(qwMsg->connection, code)); - QW_TASK_DLOG("query msg rsped, code:%d", code); + QW_ERR_JRET(qwBuildAndSendQueryRsp(&qwMsg->connInfo, code)); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); queryRsped = true; @@ -1046,105 +982,96 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { _return: - if (code) { - rspCode = code; - } + input.code = code; + code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); if (!queryRsped) { - qwBuildAndSendQueryRsp(qwMsg->connection, rspCode); - QW_TASK_DLOG("query msg rsped, code:%x", rspCode); + qwBuildAndSendQueryRsp(&qwMsg->connInfo, code); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); } - input.code = rspCode; - input.taskStatus = rspCode ? JOB_TASK_STATUS_FAILED : JOB_TASK_STATUS_PARTIAL_SUCCEED; - - QW_ERR_RET(qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); - - QW_RET(rspCode); + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; SQWTaskCtx *ctx = NULL; int8_t phase = 0; - bool needRsp = false; - int32_t rspCode = 0; + bool needRsp = true; QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || - QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task already cancelled/dropped, phase:%d", phase); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG_E("task is dropping or already dropped"); + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - - phase = QW_GET_PHASE(ctx); - if (phase == QW_PHASE_PRE_QUERY) { + if (ctx->phase == QW_PHASE_PRE_QUERY) { + ctx->connInfo.handle == qwMsg->connInfo.handle; + ctx->connInfo.ahandle = qwMsg->connInfo.ahandle; QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_READY); - ctx->readyConnection = qwMsg->connection; - QW_TASK_DLOG("ready msg not rsped, phase:%d", phase); - } else if (phase == QW_PHASE_POST_QUERY) { - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - needRsp = true; - rspCode = ctx->rspCode; - } else { - QW_TASK_ELOG("invalid phase when got ready msg, phase:%d", phase); - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - needRsp = true; - rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); + needRsp = false; + QW_TASK_DLOG_E("ready msg will not rsp now"); + goto _return; + } + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); + + if (atomic_load_8((int8_t*)&ctx->queryEnd) || atomic_load_8((int8_t*)&ctx->queryFetched)) { + QW_TASK_ELOG("got ready msg at wrong status, queryEnd:%d, queryFetched:%d", atomic_load_8((int8_t*)&ctx->queryEnd), atomic_load_8((int8_t*)&ctx->queryFetched)); + QW_ERR_JRET(TSDB_CODE_QW_MSG_ERROR); + } + + if (ctx->phase == QW_PHASE_POST_QUERY) { + code = ctx->rspCode; + goto _return; } + QW_TASK_ELOG("invalid phase when got ready msg, phase:%s", qwPhaseStr(ctx->phase)); + + QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); + _return: if (code && ctx) { QW_UPDATE_RSP_CODE(ctx, code); } + if (code) { + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAILED); + } + if (ctx) { QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); } if (needRsp) { - qwBuildAndSendReadyRsp(qwMsg->connection, rspCode); - QW_TASK_DLOG("ready msg rsped, code:%x", rspCode); + qwBuildAndSendReadyRsp(&qwMsg->connInfo, code); + QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); } - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { SQWTaskCtx *ctx = NULL; int32_t code = 0; - bool queryRsped = false; - bool needStop = false; - struct SSubplan *plan = NULL; SQWPhaseInput input = {0}; - SQWPhaseOutput output = {0}; void *rsp = NULL; int32_t dataLen = 0; bool queryEnd = false; do { - QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); - - needStop = output.needStop; - code = output.rspCode; - - if (needStop) { - QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_CQUERY); - QW_ERR_JRET(code); - } + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - atomic_store_8(&ctx->queryInQueue, 0); - atomic_store_8(&ctx->queryContinue, 0); + atomic_store_8((int8_t*)&ctx->queryInQueue, 0); + atomic_store_8((int8_t*)&ctx->queryContinue, 0); QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd)); @@ -1153,30 +1080,27 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { - QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); + QW_TASK_DLOG("task not end and buf is %s, need to continue query", qwBufStatusStr(sOutput.bufStatus)); - // RC WARNING - atomic_store_8(&ctx->queryContinue, 1); + atomic_store_8((int8_t*)&ctx->queryContinue, 1); } if (rsp) { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); - + atomic_store_8((int8_t*)&ctx->queryEnd, qComplete); + + qwMsg->connInfo = ctx->connInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); - QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); + qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); } else { - atomic_store_8(&ctx->queryContinue, 1); + atomic_store_8((int8_t*)&ctx->queryContinue, 1); } } - if (queryEnd) { - needStop = true; - } - - _return: +_return: if (NULL == ctx) { break; @@ -1186,85 +1110,70 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwFreeFetchRsp(rsp); rsp = NULL; - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, 0, code); - QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, 0); + + qwMsg->connInfo = ctx->connInfo; + qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, 0, code); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); } QW_LOCK(QW_WRITE, &ctx->lock); - if (needStop || code || 0 == atomic_load_8(&ctx->queryContinue)) { + if (queryEnd || code || 0 == atomic_load_8((int8_t*)&ctx->queryContinue)) { + // Note: if necessary, fetch need to put cquery to queue again atomic_store_8(&ctx->phase, 0); QW_UNLOCK(QW_WRITE,&ctx->lock); break; } - QW_UNLOCK(QW_WRITE,&ctx->lock); } while (true); input.code = code; - qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); + qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL); - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; - int32_t needRsp = true; - void *data = NULL; - int32_t sinkStatus = 0; int32_t dataLen = 0; - bool queryEnd = false; - bool needStop = false; bool locked = false; SQWTaskCtx *ctx = NULL; - int8_t status = 0; void *rsp = NULL; - SQWPhaseInput input = {0}; - SQWPhaseOutput output = {0}; - QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, &output)); - - needStop = output.needStop; - code = output.rspCode; - - if (needStop) { - QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_FETCH); - QW_ERR_JRET(code); - } + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - + SOutputData sOutput = {0}; QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); if (NULL == rsp) { + atomic_store_ptr(&ctx->connInfo.handle, qwMsg->connInfo.handle); + atomic_store_ptr(&ctx->connInfo.ahandle, qwMsg->connInfo.ahandle); + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); } else { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + atomic_store_8((int8_t*)&ctx->queryEnd, qComplete); } if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { - QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); + QW_TASK_DLOG("task not end and buf is %s, need to continue query", qwBufStatusStr(sOutput.bufStatus)); QW_LOCK(QW_WRITE, &ctx->lock); locked = true; // RC WARNING if (QW_IS_QUERY_RUNNING(ctx)) { - atomic_store_8(&ctx->queryContinue, 1); - } else if (0 == atomic_load_8(&ctx->queryInQueue)) { - if (!ctx->multiExec) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); - ctx->multiExec = true; - } + atomic_store_8((int8_t*)&ctx->queryContinue, 1); + } else if (0 == atomic_load_8((int8_t*)&ctx->queryInQueue)) { + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING); - atomic_store_8(&ctx->queryInQueue, 1); + atomic_store_8((int8_t*)&ctx->queryInQueue, 1); - QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); - - QW_TASK_DLOG("schedule query in queue, phase:%d", ctx->phase); + QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), &qwMsg->connInfo)); } } @@ -1275,34 +1184,31 @@ _return: } input.code = code; - - qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, &output); - - if (output.rspCode) { - code = output.rspCode; - } + code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, NULL); if (code) { qwFreeFetchRsp(rsp); rsp = NULL; dataLen = 0; - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); - QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); - } else if (rsp) { - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); - QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); } - QW_RET(code); + if (code || rsp) { + qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); + } + + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; - bool needRsp = false; + bool rsped = false; SQWTaskCtx *ctx = NULL; bool locked = false; + // TODO : TASK ALREADY REMOVED AND A NEW DROP MSG RECEIVED + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -1310,26 +1216,26 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { locked = true; if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task already dropping, phase:%d", ctx->phase); + QW_TASK_WLOG_E("task already dropping"); QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); } if (QW_IS_QUERY_RUNNING(ctx)) { QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING); } else if (ctx->phase > 0) { - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); - - QW_SET_RSP_CODE(ctx, TSDB_CODE_QRY_TASK_DROPPED); - - locked = false; - needRsp = true; + qwBuildAndSendDropRsp(&qwMsg->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); + + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + rsped = true; + } else { + // task not started } - if (!needRsp) { - ctx->dropConnection = qwMsg->connection; + if (!rsped) { + ctx->connInfo.handle = qwMsg->connInfo.handle; + ctx->connInfo.ahandle = qwMsg->connInfo.ahandle; QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } @@ -1337,7 +1243,11 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { _return: if (code) { - QW_UPDATE_RSP_CODE(ctx, code); + if (ctx) { + QW_UPDATE_RSP_CODE(ctx, code); + } + + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAILED); } if (locked) { @@ -1348,13 +1258,12 @@ _return: qwReleaseTaskCtx(mgmt, ctx); } - if (TSDB_CODE_SUCCESS != code || needRsp) { - QW_ERR_RET(qwBuildAndSendDropRsp(qwMsg->connection, code)); - - QW_TASK_DLOG("drop msg rsped, code:%x", code); + if (TSDB_CODE_SUCCESS != code) { + qwBuildAndSendDropRsp(&qwMsg->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); } - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { @@ -1362,38 +1271,46 @@ int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { SSchedulerHbRsp rsp = {0}; SQWSchStatus *sch = NULL; uint64_t seqId = 0; + void *origHandle = NULL; memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); QW_ERR_JRET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); - atomic_store_ptr(&sch->hbConnection, qwMsg->connection); - ++sch->hbSeqId; + QW_LOCK(QW_WRITE, &sch->hbConnLock); - rsp.seqId = sch->hbSeqId; - - QW_DLOG("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, connection:%p", - sch->hbSeqId, req->sId, req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connection); + if (sch->hbConnInfo.handle) { + rpcReleaseHandle(sch->hbConnInfo.handle, TAOS_CONN_SERVER); + } + + memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); + memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); + + QW_UNLOCK(QW_WRITE, &sch->hbConnLock); + + QW_DLOG("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, handle:%p, ahandle:%p", + req->sId, req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connInfo.handle, qwMsg->connInfo.ahandle); qwReleaseScheduler(QW_READ, mgmt); _return: - qwBuildAndSendHbRsp(qwMsg->connection, &rsp, code); + qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); + QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } void qwProcessHbTimerEvent(void *param, void *tmrId) { - return; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)param; SQWSchStatus *sch = NULL; int32_t taskNum = 0; SQWHbInfo *rspList = NULL; int32_t code = 0; + qwDbgDumpMgmtInfo(mgmt); + QW_LOCK(QW_READ, &mgmt->schLock); int32_t schNum = taosHashGetSize(mgmt->schHash); @@ -1403,7 +1320,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { return; } - rspList = calloc(schNum, sizeof(SQWHbInfo)); + rspList = taosMemoryCalloc(schNum, sizeof(SQWHbInfo)); if (NULL == rspList) { QW_UNLOCK(QW_READ, &mgmt->schLock); QW_ELOG("calloc %d SQWHbInfo failed", schNum); @@ -1432,25 +1349,24 @@ _return: QW_UNLOCK(QW_READ, &mgmt->schLock); for (int32_t j = 0; j < i; ++j) { - QW_DLOG("hb on connection %p, taskNum:%d", rspList[j].connection, (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0)); - qwBuildAndSendHbRsp(rspList[j].connection, &rspList[j].rsp, code); + qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); + QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0)); tFreeSSchedulerHbRsp(&rspList[j].rsp); } - tfree(rspList); + taosMemoryFreeClear(rspList); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); } -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, - putReqToQueryQFp fp1, sendReqToDnodeFp fp2) { - if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp1 || NULL == fp2) { +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb) { + if (NULL == qWorkerMgmt || pMsgCb->pWrapper == NULL) { qError("invalid param to init qworker"); QW_RET(TSDB_CODE_QRY_INVALID_INPUT); } int32_t code = 0; - SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); + SQWorkerMgmt *mgmt = taosMemoryCalloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1475,7 +1391,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == mgmt->schHash) { - tfree(mgmt); + taosMemoryFreeClear(mgmt); qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1500,9 +1416,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->nodeType = nodeType; mgmt->nodeId = nodeId; - mgmt->nodeObj = nodeObj; - mgmt->putToQueueFp = fp1; - mgmt->sendReqFp = fp2; + mgmt->msgCb = *pMsgCb; *qWorkerMgmt = mgmt; @@ -1517,7 +1431,7 @@ _return: taosTmrCleanUp(mgmt->timer); - tfree(mgmt); + taosMemoryFreeClear(mgmt); QW_RET(code); } @@ -1536,7 +1450,7 @@ void qWorkerDestroy(void **qWorkerMgmt) { //TODO FREE ALL - tfree(*qWorkerMgmt); + taosMemoryFreeClear(*qWorkerMgmt); } int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { @@ -1553,7 +1467,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs taskNum = taosHashGetSize(sch->tasksHash); int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; - *rsp = calloc(1, size); + *rsp = taosMemoryCalloc(1, size); if (NULL == *rsp) { QW_SCH_ELOG("calloc %d failed", size); QW_UNLOCK(QW_READ, &sch->tasksLock); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 7d633d1c731480122e3b737ea1c18d336723de2e..dd1f2002728aa831d8640a694687896401775a48 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -26,6 +26,8 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { return TSDB_CODE_SUCCESS; } + + void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; @@ -44,17 +46,19 @@ void qwFreeFetchRsp(void *msg) { } } -int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) { - SRpcMsg *pMsg = (SRpcMsg *)connection; - SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp)); - pRsp->code = code; +int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) { + SQueryTableRsp rsp = {.code = code}; + + int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp); + void *msg = rpcMallocCont(contLen); + tSerializeSQueryTableRsp(msg, contLen, &rsp); SRpcMsg rpcRsp = { .msgType = TDMT_VND_QUERY_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, - .pCont = pRsp, - .contLen = sizeof(*pRsp), + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .pCont = msg, + .contLen = contLen, .code = code, }; @@ -63,15 +67,14 @@ int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) { - SRpcMsg *pMsg = (SRpcMsg *)connection; +int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) { SResReadyRsp *pRsp = (SResReadyRsp *)rpcMallocCont(sizeof(SResReadyRsp)); pRsp->code = code; SRpcMsg rpcRsp = { .msgType = TDMT_VND_RES_READY_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = NULL, .pCont = pRsp, .contLen = sizeof(*pRsp), .code = code, @@ -82,15 +85,15 @@ int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *pStatus, int32_t code) { +int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) { int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus); void *pRsp = rpcMallocCont(contLen); tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus); SRpcMsg rpcRsp = { .msgType = TDMT_VND_QUERY_HEARTBEAT_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = pRsp, .contLen = contLen, .code = code, @@ -101,9 +104,7 @@ int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *pStatus, int32_t cod return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) { - SRpcMsg *pMsg = (SRpcMsg *)connection; - +int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) { if (NULL == pRsp) { pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRsp, 0, sizeof(SRetrieveTableRsp)); @@ -112,8 +113,8 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_ SRpcMsg rpcRsp = { .msgType = TDMT_VND_FETCH_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = pRsp, .contLen = sizeof(*pRsp) + dataLength, .code = code, @@ -124,14 +125,14 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_ return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { +int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) { STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp)); pRsp->code = code; SRpcMsg rpcRsp = { .msgType = TDMT_VND_CANCEL_TASK_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = pRsp, .contLen = sizeof(*pRsp), .code = code, @@ -141,15 +142,14 @@ int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendDropRsp(void *connection, int32_t code) { - SRpcMsg *pMsg = (SRpcMsg *)connection; +int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) { STaskDropRsp *pRsp = (STaskDropRsp *)rpcMallocCont(sizeof(STaskDropRsp)); pRsp->code = code; SRpcMsg rpcRsp = { .msgType = TDMT_VND_DROP_TASK_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = pRsp, .contLen = sizeof(*pRsp), .code = code, @@ -164,7 +164,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { SVShowTablesRsp showRsp = {0}; // showRsp.showId = 1; - showRsp.tableMeta.pSchemas = calloc(numOfCols, sizeof(SSchema)); + showRsp.tableMeta.pSchemas = taosMemoryCalloc(numOfCols, sizeof(SSchema)); if (showRsp.tableMeta.pSchemas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -231,8 +231,7 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) { - SRpcMsg *pMsg = (SRpcMsg *)connection; +int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); @@ -245,26 +244,55 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) { req->taskId = tId; SRpcMsg pNewMsg = { - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .msgType = TDMT_VND_QUERY_CONTINUE, .pCont = req, .contLen = sizeof(SQueryContinueReq), .code = 0, }; - int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); + int32_t code = tmsgPutToQueue(&mgmt->msgCb, QUERY_QUEUE, &pNewMsg); if (TSDB_CODE_SUCCESS != code) { QW_SCH_TASK_ELOG("put query continue msg to queue failed, vgId:%d, code:%s", mgmt->nodeId, tstrerror(code)); rpcFreeCont(req); QW_ERR_RET(code); } - QW_SCH_TASK_DLOG("put task continue exec msg to query queue, vgId:%d", mgmt->nodeId); + QW_SCH_TASK_DLOG("query continue msg put to queue, vgId:%d", mgmt->nodeId); return TSDB_CODE_SUCCESS; } + +int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { + STaskDropReq * req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq)); + if (NULL == req) { + QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq)); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + req->header.vgId = htonl(mgmt->nodeId); + req->sId = htobe64(sId); + req->queryId = htobe64(qId); + req->taskId = htobe64(tId); + req->refId = htobe64(rId); + + SRpcMsg pMsg = { + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .msgType = TDMT_VND_DROP_TASK, + .pCont = req, + .contLen = sizeof(STaskDropReq), + .code = TSDB_CODE_RPC_NETWORK_UNAVAIL, + }; + + rpcRegisterBrokenLinkArg(&pMsg); + + return TSDB_CODE_SUCCESS; +} + + int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -291,13 +319,15 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t tId = msg->taskId; int64_t rId = msg->refId; - SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; char* sql = strndup(msg->msg, msg->sqlLen); - QW_SCH_TASK_DLOG("processQuery start, node:%p, sql:%s", node, sql); - tfree(sql); + QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->handle, sql); + taosMemoryFreeClear(sql); - QW_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); + QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); @@ -323,9 +353,11 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t tId = msg->taskId; int64_t rId = 0; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_TASK_DLOG("processCQuery start, node:%p", node); + QW_SCH_TASK_DLOG("processCQuery start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessCQuery(QW_FPARAMS(), &qwMsg)); @@ -355,9 +387,11 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ uint64_t tId = msg->taskId; int64_t rId = 0; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_TASK_DLOG("processReady start, node:%p", node); + QW_SCH_TASK_DLOG("processReady start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessReady(QW_FPARAMS(), &qwMsg)); @@ -415,9 +449,11 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t tId = msg->taskId; int64_t rId = 0; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_TASK_DLOG("processFetch start, node:%p", node); + QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessFetch(QW_FPARAMS(), &qwMsg)); @@ -436,6 +472,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; int32_t code = 0; STaskCancelReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { @@ -448,11 +485,21 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { msg->taskId = be64toh(msg->taskId); msg->refId = be64toh(msg->refId); + uint64_t sId = msg->sId; + uint64_t qId = msg->queryId; + uint64_t tId = msg->taskId; + int64_t rId = msg->refId; + + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; + //QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId)); _return: - QW_ERR_RET(qwBuildAndSendCancelRsp(pMsg, code)); + QW_ERR_RET(qwBuildAndSendCancelRsp(&qwMsg.connInfo, code)); + QW_SCH_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", qwMsg.connInfo.handle, code, tstrerror(code)); return TSDB_CODE_SUCCESS; } @@ -481,9 +528,15 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t tId = msg->taskId; int64_t rId = msg->refId; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; + + if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) { + QW_SCH_TASK_DLOG("receive drop task due to network broken, error:%s", tstrerror(pMsg->code)); + } - QW_SCH_TASK_DLOG("processDrop start, node:%p", node); + QW_SCH_TASK_DLOG("processDrop start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessDrop(QW_FPARAMS(), &qwMsg)); @@ -513,9 +566,11 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } uint64_t sId = req.sId; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_DLOG("processHb start, node:%p", node); + QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessHb(mgmt, &qwMsg, &req)); @@ -532,7 +587,7 @@ int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SVShowTablesReq *pReq = pMsg->pCont; - QW_ERR_RET(qwBuildAndSendShowRsp(pMsg, code)); + QW_RET(qwBuildAndSendShowRsp(pMsg, code)); } int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { @@ -541,7 +596,7 @@ int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) } SVShowTablesFetchReq *pFetchReq = pMsg->pCont; - QW_ERR_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq)); + QW_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq)); } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index d7814a5dfbaadabc0b5e1b7bc05e91c3084e7192..7179fa6984b7a5564fac9daf4d393b34c2c4ba7f 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -47,17 +47,19 @@ namespace { #define qwtTestQueryQueueSize 1000000 #define qwtTestFetchQueueSize 1000000 +bool qwtEnableLog = true; + int32_t qwtTestMaxExecTaskUsec = 2; int32_t qwtTestReqMaxDelayUsec = 2; -uint64_t qwtTestQueryId = 0; +int64_t qwtTestQueryId = 0; bool qwtTestEnableSleep = true; bool qwtTestStop = false; bool qwtTestDeadLoop = false; -int32_t qwtTestMTRunSec = 60; -int32_t qwtTestPrintNum = 100000; -int32_t qwtTestCaseIdx = 0; -int32_t qwtTestCaseNum = 4; +int32_t qwtTestMTRunSec = 2; +int32_t qwtTestPrintNum = 10000; +uint64_t qwtTestCaseIdx = 0; +uint64_t qwtTestCaseNum = 4; bool qwtTestCaseFinished = false; tsem_t qwtTestQuerySem; tsem_t qwtTestFetchSem; @@ -95,11 +97,15 @@ SSchTasksStatusReq qwtstatusMsg = {0}; void qwtInitLogFile() { + if (!qwtEnableLog) { + return; + } const char *defaultLogFileNamePrefix = "taosdlog"; const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -159,7 +165,7 @@ int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestFetchQueueLock); - struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg; if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { @@ -182,7 +188,7 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestQueryQueueLock); - struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg; if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { @@ -202,6 +208,9 @@ int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { return 0; } +void qwtSendReqToDnode(void* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { + +} void qwtRpcSendResponse(const SRpcMsg *pRsp) { @@ -240,6 +249,7 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { if (0 == pRsp->code && 0 == rsp->completed) { qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc); qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc); + rpcFreeCont(rsp); return; } @@ -262,26 +272,15 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { return; } -int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { - int32_t idx = abs((++qwtTestCaseIdx) % qwtTestCaseNum); - +int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { qwtTestSinkBlockNum = 0; qwtTestSinkMaxBlockNum = taosRand() % 100 + 1; qwtTestSinkQueryEnd = false; - if (0 == idx) { - *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx; - *handle = (DataSinkHandle)qwtTestCaseIdx+1; - } else if (1 == idx) { - *pTaskInfo = NULL; - *handle = NULL; - } else if (2 == idx) { - *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx; - *handle = NULL; - } else if (3 == idx) { - *pTaskInfo = NULL; - *handle = (DataSinkHandle)qwtTestCaseIdx; - } + *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx+1; + *handle = (DataSinkHandle)qwtTestCaseIdx+2; + + ++qwtTestCaseIdx; return 0; } @@ -313,8 +312,8 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { } if (endExec) { - *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock)); - (*pRes)->info.rows = taosRand() % 1000; + *pRes = (SSDataBlock*)taosMemoryCalloc(1, sizeof(SSDataBlock)); + (*pRes)->info.rows = taosRand() % 1000 + 1; } else { *pRes = NULL; *useconds = taosRand() % 10; @@ -337,7 +336,7 @@ int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* p assert(0); } - free((void *)pInput->pData); + taosMemoryFree((void *)pInput->pData); taosWLockLatch(&qwtTestSinkLock); @@ -764,7 +763,7 @@ void *queryQueueThread(void *param) { assert(0); } - free(queryRpc); + taosMemoryFree(queryRpc); if (qwtTestStop && qwtTestQueryQueueNum <= 0 && qwtTestCaseFinished) { break; @@ -833,7 +832,7 @@ void *fetchQueueThread(void *param) { break; } - free(fetchRpc); + taosMemoryFree(fetchRpc); if (qwtTestStop && qwtTestFetchQueueNum <= 0 && qwtTestCaseFinished) { break; @@ -849,7 +848,6 @@ void *fetchQueueThread(void *param) { } -#if 0 TEST(seqTest, normalCase) { void *mgmt = NULL; @@ -880,14 +878,17 @@ TEST(seqTest, normalCase) { stubSetPutDataBlock(); stubSetGetDataBlock(); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); ASSERT_EQ(code, 0); - code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); - ASSERT_EQ(code, 0); + //code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + //ASSERT_EQ(code, 0); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); ASSERT_EQ(code, 0); @@ -919,7 +920,10 @@ TEST(seqTest, cancelFirst) { stubSetStringToPlan(); stubSetRpcSendResponse(); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); @@ -965,7 +969,10 @@ TEST(seqTest, randCase) { taosSeedRand(taosGetTimestampSec()); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); int32_t t = 0; @@ -978,12 +985,12 @@ TEST(seqTest, randCase) { qwtBuildQueryReqMsg(&queryRpc); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); } else if (r >= maxr/5 && r < maxr * 2/5) { - printf("Ready,%d\n", t++); - qwtBuildReadyReqMsg(&readyMsg, &readyRpc); - code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); - if (qwtTestEnableSleep) { - taosUsleep(1); - } + //printf("Ready,%d\n", t++); + //qwtBuildReadyReqMsg(&readyMsg, &readyRpc); + //code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + //if (qwtTestEnableSleep) { + // taosUsleep(1); + //} } else if (r >= maxr * 2/5 && r < maxr* 3/5) { printf("Fetch,%d\n", t++); qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); @@ -1024,21 +1031,34 @@ TEST(seqTest, multithreadRand) { stubSetStringToPlan(); stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); taosSeedRand(taosGetTimestampSec()); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, queryThread, mgmt); - pthread_create(&(t2), &thattr, readyThread, NULL); - pthread_create(&(t3), &thattr, fetchThread, NULL); - pthread_create(&(t4), &thattr, dropThread, NULL); - pthread_create(&(t5), &thattr, statusThread, NULL); + TdThread t1,t2,t3,t4,t5,t6; + taosThreadCreate(&(t1), &thattr, queryThread, mgmt); + //taosThreadCreate(&(t2), &thattr, readyThread, NULL); + taosThreadCreate(&(t3), &thattr, fetchThread, NULL); + taosThreadCreate(&(t4), &thattr, dropThread, NULL); + taosThreadCreate(&(t5), &thattr, statusThread, NULL); + taosThreadCreate(&(t6), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1051,12 +1071,19 @@ TEST(seqTest, multithreadRand) { qwtTestStop = true; taosSsleep(3); + + qwtTestQueryQueueNum = 0; + qwtTestQueryQueueRIdx = 0; + qwtTestQueryQueueWIdx = 0; + qwtTestQueryQueueLock = 0; + qwtTestFetchQueueNum = 0; + qwtTestFetchQueueRIdx = 0; + qwtTestFetchQueueWIdx = 0; + qwtTestFetchQueueLock = 0; qWorkerDestroy(&mgmt); } -#endif - TEST(rcTest, shortExecshortDelay) { void *mgmt = NULL; int32_t code = 0; @@ -1080,7 +1107,10 @@ TEST(rcTest, shortExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1089,13 +1119,13 @@ TEST(rcTest, shortExecshortDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1161,7 +1191,10 @@ TEST(rcTest, longExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 1000000; @@ -1170,13 +1203,13 @@ TEST(rcTest, longExecshortDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1244,7 +1277,10 @@ TEST(rcTest, shortExeclongDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1253,13 +1289,13 @@ TEST(rcTest, shortExeclongDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1304,7 +1340,6 @@ TEST(rcTest, shortExeclongDelay) { } -#if 0 TEST(rcTest, dropTest) { void *mgmt = NULL; int32_t code = 0; @@ -1326,19 +1361,22 @@ TEST(rcTest, dropTest) { taosSeedRand(taosGetTimestampSec()); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, clientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1354,7 +1392,6 @@ TEST(rcTest, dropTest) { qWorkerDestroy(&mgmt); } -#endif int main(int argc, char** argv) { diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index e1ffb2efd12cdd00f8cdd59de5c60576239c34c8..834a722bd80e6f8374adf7a3c2ac685c843a61fb 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -338,7 +338,7 @@ typedef struct SFilterInfo { #define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) #define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) -#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) +#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = taosMemoryMalloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) diff --git a/source/libs/scalar/inc/sclfunc.h b/source/libs/scalar/inc/sclfunc.h index 8e034700338ff5c22943a5f78a816e1085caeed8..8915f37261713ebe08d4262288efa1557f204338 100644 --- a/source/libs/scalar/inc/sclfunc.h +++ b/source/libs/scalar/inc/sclfunc.h @@ -37,7 +37,6 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); - #ifdef __cplusplus } #endif diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 58b5c8340a2152d3335d411be2d2cd58f0fd0caf..23e3e352c86622ab64cf1a27004247355edf7ac9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -297,14 +297,14 @@ static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void int32_t filterInitUnitsFields(SFilterInfo *info) { info->unitSize = FILTER_DEFAULT_UNIT_SIZE; - info->units = calloc(info->unitSize, sizeof(SFilterUnit)); + info->units = taosMemoryCalloc(info->unitSize, sizeof(SFilterUnit)); info->fields[FLD_TYPE_COLUMN].num = 0; info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); + info->fields[FLD_TYPE_COLUMN].fields = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); info->fields[FLD_TYPE_VALUE].num = 0; info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); + info->fields[FLD_TYPE_VALUE].fields = taosMemoryCalloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); return TSDB_CODE_SUCCESS; } @@ -318,7 +318,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt r->prev = NULL; r->next = NULL; } else { - r = calloc(1, sizeof(SFilterRangeNode)); + r = taosMemoryCalloc(1, sizeof(SFilterRangeNode)); } FILTER_COPY_RA(&r->ra, ra); @@ -332,7 +332,7 @@ void* filterInitRangeCtx(int32_t type, int32_t options) { return NULL; } - SFilterRangeCtx *ctx = calloc(1, sizeof(SFilterRangeCtx)); + SFilterRangeCtx *ctx = taosMemoryCalloc(1, sizeof(SFilterRangeCtx)); ctx->type = type; ctx->options = options; @@ -748,18 +748,18 @@ int32_t filterFreeRangeCtx(void* h) { while (r) { rn = r->next; - free(r); + taosMemoryFree(r); r = rn; } r = ctx->rf; while (r) { rn = r->next; - free(r); + taosMemoryFree(r); r = rn; } - free(ctx); + taosMemoryFree(ctx); return TSDB_CODE_SUCCESS; } @@ -769,7 +769,7 @@ int32_t filterDetachCnfGroup(SFilterGroup *gp1, SFilterGroup *gp2, SArray* group SFilterGroup gp = {0}; gp.unitNum = gp1->unitNum + gp2->unitNum; - gp.unitIdxs = calloc(gp.unitNum, sizeof(*gp.unitIdxs)); + gp.unitIdxs = taosMemoryCalloc(gp.unitNum, sizeof(*gp.unitIdxs)); memcpy(gp.unitIdxs, gp1->unitIdxs, gp1->unitNum * sizeof(*gp.unitIdxs)); memcpy(gp.unitIdxs + gp1->unitNum, gp2->unitIdxs, gp2->unitNum * sizeof(*gp.unitIdxs)); @@ -849,7 +849,7 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d return -1; } -// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0 +// In the params, we should use void *data instead of void **data, there is no need to use taosMemoryFreeClear(*data) to set *data = 0 // Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better. int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { int32_t idx = -1; @@ -869,7 +869,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, idx = *num; if (idx >= info->fields[type].size) { info->fields[type].size += FILTER_DEFAULT_FIELD_SIZE; - info->fields[type].fields = realloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); + info->fields[type].fields = taosMemoryRealloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); } info->fields[type].fields[idx].flag = type; @@ -891,7 +891,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, } } else { if (data && freeIfExists) { - tfree(*data); + taosMemoryFreeClear(*data); } } @@ -954,7 +954,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi if (info->unitNum >= info->unitSize) { uint32_t psize = info->unitSize; info->unitSize += FILTER_DEFAULT_UNIT_SIZE; - info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); + info->units = taosMemoryRealloc(info->units, info->unitSize * sizeof(SFilterUnit)); memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); } @@ -1000,7 +1000,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) { if (group->unitNum >= group->unitSize) { group->unitSize += FILTER_DEFAULT_UNIT_SIZE; - group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); + group->unitIdxs = taosMemoryRealloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); } group->unitIdxs[group->unitNum++] = unitIdx; @@ -1028,12 +1028,12 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { in.type = valueNode->node.resType.type; in.bytes = valueNode->node.resType.bytes; in.data = nodesGetValueFromNode(valueNode); - out.data = malloc(sizeof(int64_t)); + out.data = taosMemoryMalloc(sizeof(int64_t)); code = vectorConvertImpl(&in, &out); if (code) { fltError("convert from %d to %d failed", in.type, out.type); - tfree(out.data); + taosMemoryFreeClear(out.data); FLT_ERR_RET(code); } @@ -1148,17 +1148,17 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); if (func(&ra->s, &ra->e) == 0) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } else { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); + void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &ra->e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); @@ -1170,7 +1170,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); @@ -1178,7 +1178,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); @@ -1224,16 +1224,16 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) &&(!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); if (func(&r->ra.s, &r->ra.e) == 0) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } else { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); + void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &r->ra.e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); @@ -1250,7 +1250,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); @@ -1258,7 +1258,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); @@ -1284,8 +1284,8 @@ static void filterFreeGroup(void *pItem) { } SFilterGroup* p = (SFilterGroup*) pItem; - tfree(p->unitIdxs); - tfree(p->unitFlags); + taosMemoryFreeClear(p->unitIdxs); + taosMemoryFreeClear(p->unitFlags); } @@ -1635,7 +1635,7 @@ void filterFreeGroupCtx(SFilterGroupCtx* gRes) { return; } - tfree(gRes->colIdx); + taosMemoryFreeClear(gRes->colIdx); int16_t i = 0, j = 0; @@ -1648,8 +1648,8 @@ void filterFreeGroupCtx(SFilterGroupCtx* gRes) { ++j; } - tfree(gRes->colInfo); - tfree(gRes); + taosMemoryFreeClear(gRes->colInfo); + taosMemoryFreeClear(gRes); } void filterFreeField(SFilterField* field, int32_t type) { @@ -1661,7 +1661,7 @@ void filterFreeField(SFilterField* field, int32_t type) { if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { taosHashCleanup(field->data); } else { - tfree(field->data); + taosMemoryFreeClear(field->data); } } } @@ -1676,40 +1676,40 @@ void filterFreeInfo(SFilterInfo *info) { return; } - tfree(info->cunits); - tfree(info->blkUnitRes); - tfree(info->blkUnits); + taosMemoryFreeClear(info->cunits); + taosMemoryFreeClear(info->blkUnitRes); + taosMemoryFreeClear(info->blkUnits); for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); } - tfree(info->fields[i].fields); + taosMemoryFreeClear(info->fields[i].fields); } for (uint32_t i = 0; i < info->groupNum; ++i) { filterFreeGroup(&info->groups[i]); } - tfree(info->groups); + taosMemoryFreeClear(info->groups); - tfree(info->units); + taosMemoryFreeClear(info->units); - tfree(info->unitRes); + taosMemoryFreeClear(info->unitRes); - tfree(info->unitFlags); + taosMemoryFreeClear(info->unitFlags); for (uint32_t i = 0; i < info->colRangeNum; ++i) { filterFreeRangeCtx(info->colRange[i]); } - tfree(info->colRange); + taosMemoryFreeClear(info->colRange); filterFreePCtx(&info->pctx); if (!FILTER_GET_FLAG(info->status, FI_STATUS_CLONED)) { - tfree(info); + taosMemoryFreeClear(info); } } @@ -1775,14 +1775,14 @@ int32_t fltInitValFieldData(SFilterInfo *info) { if (type == TSDB_DATA_TYPE_BINARY) { size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; - fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); + fi->data = taosMemoryCalloc(1, len + 1 + VARSTR_HEADER_SIZE); } else if (type == TSDB_DATA_TYPE_NCHAR) { size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; - fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + fi->data = taosMemoryCalloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); } else if (type != TSDB_DATA_TYPE_JSON){ if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE /* - fi->data = calloc(dType->bytes, tDataTypes[type].bytes); + fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes); for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); @@ -1790,7 +1790,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { */ continue; } else { - fi->data = calloc(1, sizeof(int64_t)); + fi->data = taosMemoryCalloc(1, sizeof(int64_t)); } } else{ // type == TSDB_DATA_TYPE_JSON // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) @@ -1998,15 +1998,15 @@ _return: int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { bool empty = false; - uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); + uint32_t *colIdx = taosMemoryMalloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); uint32_t colIdxi = 0; uint32_t gResIdx = 0; for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup* g = info->groups + i; - gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); - gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); + gRes[gResIdx] = taosMemoryCalloc(1, sizeof(SFilterGroupCtx)); + gRes[gResIdx]->colInfo = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); colIdxi = 0; empty = false; @@ -2058,7 +2058,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t ++gResIdx; } - tfree(colIdx); + taosMemoryFreeClear(colIdx); *gResNum = gResIdx; @@ -2350,12 +2350,12 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { info->groupNum = (uint32_t)groupSize; if (info->groupNum > 0) { - info->groups = calloc(info->groupNum, sizeof(*info->groups)); + info->groups = taosMemoryCalloc(info->groupNum, sizeof(*info->groups)); } for (size_t i = 0; i < groupSize; ++i) { SFilterGroup *pg = taosArrayGet(group, i); - pg->unitFlags = calloc(pg->unitNum, sizeof(*pg->unitFlags)); + pg->unitFlags = taosMemoryCalloc(pg->unitNum, sizeof(*pg->unitFlags)); info->groups[i] = *pg; } @@ -2435,7 +2435,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ uint32_t *idxs = NULL; uint32_t colNum = 0; SFilterGroupCtx *res = NULL; - uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + uint32_t *idxNum = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); for (int32_t i = 0; i < gResNum; ++i) { for (uint32_t m = 0; m < gRes[i]->colNum; ++m) { @@ -2456,7 +2456,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ assert(idxNum[i] == gResNum); if (idxs == NULL) { - idxs = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); + idxs = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); } idxs[colNum++] = i; @@ -2465,7 +2465,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ FLT_CHK_JMP(colNum <= 0); info->colRangeNum = colNum; - info->colRange = calloc(colNum, POINTER_BYTES); + info->colRange = taosMemoryCalloc(colNum, POINTER_BYTES); for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; @@ -2512,8 +2512,8 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ } _return: - tfree(idxNum); - tfree(idxs); + taosMemoryFreeClear(idxNum); + taosMemoryFreeClear(idxs); return TSDB_CODE_SUCCESS; } @@ -2533,9 +2533,9 @@ int32_t filterPostProcessRange(SFilterInfo *info) { int32_t filterGenerateComInfo(SFilterInfo *info) { - info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); - info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); - info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); + info->cunits = taosMemoryMalloc(info->unitNum * sizeof(*info->cunits)); + info->blkUnitRes = taosMemoryMalloc(sizeof(*info->blkUnitRes) * info->unitNum); + info->blkUnits = taosMemoryMalloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; @@ -2766,7 +2766,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint32_t *unitIdx = NULL; if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2871,7 +2871,7 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2905,7 +2905,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2947,7 +2947,7 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2977,7 +2977,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2991,7 +2991,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa // match/nmatch for nchar type need convert from ucs4 to mbs if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){ - char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + char *newColData = taosMemoryCalloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ qError("castConvert1 taosUcs4ToMbs error"); @@ -2999,7 +2999,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa varDataSetLen(newColData, len); (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); } - tfree(newColData); + taosMemoryFreeClear(newColData); }else{ (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); } @@ -3022,7 +3022,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -3051,7 +3051,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ - char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + char *newColData = taosMemoryCalloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ qError("castConvert1 taosUcs4ToMbs error"); @@ -3059,7 +3059,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg varDataSetLen(newColData, len); (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); } - tfree(newColData); + taosMemoryFreeClear(newColData); }else{ (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } @@ -3125,7 +3125,7 @@ int32_t filterSetExecFunc(SFilterInfo *info) { int32_t filterPreprocess(SFilterInfo *info) { - SFilterGroupCtx** gRes = calloc(info->groupNum, sizeof(SFilterGroupCtx *)); + SFilterGroupCtx** gRes = taosMemoryCalloc(info->groupNum, sizeof(SFilterGroupCtx *)); int32_t gResNum = 0; filterMergeGroupUnits(info, gRes, &gResNum); @@ -3161,7 +3161,7 @@ _return: filterFreeGroupCtx(gRes[i]); } - tfree(gRes); + taosMemoryFreeClear(gRes); return TSDB_CODE_SUCCESS; } @@ -3216,8 +3216,8 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { } } - info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); - info->unitFlags = malloc(info->unitNum * sizeof(*info->unitFlags)); + info->unitRes = taosMemoryMalloc(info->unitNum * sizeof(*info->unitRes)); + info->unitFlags = taosMemoryMalloc(info->unitNum * sizeof(*info->unitFlags)); filterDumpInfoToString(info, "Final", 0); @@ -3418,7 +3418,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar SFilterField nfi = {0}; nfi.desc = fi->desc; int32_t bytes = FILTER_GET_COL_FIELD_SIZE(fi); - nfi.data = malloc(rows * bytes); + nfi.data = taosMemoryMalloc(rows * bytes); int32_t bufSize = bytes - VARSTR_HEADER_SIZE; for (int32_t j = 0; j < rows; ++j) { char *src = FILTER_GET_COL_FIELD_DATA(fi, j); @@ -3459,7 +3459,7 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { - tfree(fi->data); + taosMemoryFreeClear(fi->data); } } @@ -3634,7 +3634,7 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) } if (*pInfo == NULL) { - *pInfo = calloc(1, sizeof(SFilterInfo)); + *pInfo = taosMemoryCalloc(1, sizeof(SFilterInfo)); if (NULL == *pInfo) { fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo)); FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 8e7eaa0f8c65818504de1eb507f86376aa7dd077..83902c9df856b53cb096e858a480bd09c25ae006 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -31,7 +31,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; int8_t dummy = 0; int32_t bufLen = 60; - out.data = malloc(bufLen); + out.data = taosMemoryMalloc(bufLen); int32_t len = 0; void *buf = NULL; @@ -75,14 +75,14 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { cell = cell->pNext; } - tfree(out.data); + taosMemoryFreeClear(out.data); *data = pObj; return TSDB_CODE_SUCCESS; _return: - tfree(out.data); + taosMemoryFreeClear(out.data); taosHashCleanup(pObj); SCL_RET(code); @@ -98,7 +98,7 @@ FORCE_INLINE bool sclIsNull(SScalarParam* param, int32_t idx) { FORCE_INLINE void sclSetNull(SScalarParam* param, int32_t idx) { if (NULL == param->bitmap) { - param->bitmap = calloc(BitmapLen(param->num), sizeof(char)); + param->bitmap = taosMemoryCalloc(BitmapLen(param->num), sizeof(char)); if (NULL == param->bitmap) { sclError("calloc %d failed", param->num); return; @@ -126,7 +126,7 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParamNoData(SScalarParam *param) { - tfree(param->bitmap); + taosMemoryFreeClear(param->bitmap); } @@ -137,7 +137,7 @@ void sclFreeParam(SScalarParam *param) { if (SCL_DATA_TYPE_DUMMY_HASH == param->type) { taosHashCleanup((SHashObj *)param->orig.data); } else { - tfree(param->orig.data); + taosMemoryFreeClear(param->orig.data); } } } @@ -148,7 +148,7 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { return TSDB_CODE_SUCCESS; } - *res = malloc(pNode->node.resType.bytes); + *res = taosMemoryMalloc(pNode->node.resType.bytes); if (NULL == (*res)) { sclError("malloc %d failed", pNode->node.resType.bytes); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -284,7 +284,7 @@ int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx) int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { int32_t code = 0; - SScalarParam *paramList = calloc(pParamList->length, sizeof(SScalarParam)); + SScalarParam *paramList = taosMemoryCalloc(pParamList->length, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -308,7 +308,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarC _return: - tfree(paramList); + taosMemoryFreeClear(paramList); SCL_RET(code); } @@ -320,7 +320,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SScalarParam *paramList = calloc(paramNum, sizeof(SScalarParam)); + SScalarParam *paramList = taosMemoryCalloc(paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -337,7 +337,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal _return: - tfree(paramList); + taosMemoryFreeClear(paramList); SCL_RET(code); } @@ -360,7 +360,7 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); output->type = node->node.resType.type; - output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); + output->data = taosMemoryCalloc(rowNum, sizeof(tDataTypes[output->type].bytes)); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -384,7 +384,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } @@ -415,7 +415,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->type = node->node.resType.type; output->bytes = sizeof(bool); output->num = rowNum; - output->data = calloc(rowNum, sizeof(bool)); + output->data = taosMemoryCalloc(rowNum, sizeof(bool)); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -449,7 +449,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } @@ -463,7 +463,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp output->type = node->node.resType.type; output->num = rowNum; output->bytes = tDataTypes[output->type].bytes; - output->data = calloc(rowNum, tDataTypes[output->type].bytes); + output->data = taosMemoryCalloc(rowNum, tDataTypes[output->type].bytes); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -485,7 +485,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index fd900afcda78a255cde001ffba8c29d69b826b36..f2fdb29e4a2c7b490ce13b0361f88559b9e6ff50 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -4,145 +4,536 @@ static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { dst->type = src->type; dst->bytes = src->bytes; - dst->num = src->num; + //dst->num = src->num; } -static void tceil(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assignBasicParaInfo(pOutput, pLeft); - assert(numOfInput == 1); +/** Math functions **/ +int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + assignBasicParaInfo(pOutput, pInput); + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = ceilf(p[i]); + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + switch (pInput->type) { + case TSDB_DATA_TYPE_FLOAT: { + float v; + GET_TYPED_DATA(v, float, pInput->type, input); + float result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_TINYINT: { + int8_t v; + GET_TYPED_DATA(v, int8_t, pInput->type, input); + int8_t result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t v; + GET_TYPED_DATA(v, int16_t, pInput->type, input); + int16_t result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_INT: { + int32_t v; + GET_TYPED_DATA(v, int32_t, pInput->type, input); + int32_t result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t v; + GET_TYPED_DATA(v, int64_t, pInput->type, input); + int64_t result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + default: { + memcpy(output, input, pInput->bytes); + break; } } + } - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*)pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = ceil(p[i]); + return TSDB_CODE_SUCCESS; +} + +int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 || !IS_NUMERIC_TYPE(pInput[0].type) || !IS_NUMERIC_TYPE(pInput[1].type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char **input = NULL, *output = NULL; + bool hasNullInput = false; + input = taosMemoryCalloc(inputNum, sizeof(char *)); + for (int32_t i = 0; i < pOutput->num; ++i) { + for (int32_t j = 0; j < inputNum; ++j) { + if (pInput[j].num == 1) { + input[j] = pInput[j].data; + } else { + input[j] = pInput[j].data + i * pInput[j].bytes; + } + if (isNull(input[j], pInput[j].type)) { + hasNullInput = true; + break; } } + output = pOutput->data + i * pOutput->bytes; - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + if (hasNullInput) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double base; + GET_TYPED_DATA(base, double, pInput[1].type, input[1]); + double v; + GET_TYPED_DATA(v, double, pInput[0].type, input[0]); + double result = log(v) / log(base); + SET_TYPED_DATA(output, pOutput->type, result); } -} -static void tfloor(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assignBasicParaInfo(pOutput, pLeft); - assert(numOfInput == 1); + taosMemoryFree(input); - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; + return TSDB_CODE_SUCCESS; +} + +int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 || !IS_NUMERIC_TYPE(pInput[0].type) || !IS_NUMERIC_TYPE(pInput[1].type)) { + return TSDB_CODE_FAILED; + } - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = floorf(p[i]); + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char **input = NULL, *output = NULL; + bool hasNullInput = false; + input = taosMemoryCalloc(inputNum, sizeof(char *)); + for (int32_t i = 0; i < pOutput->num; ++i) { + for (int32_t j = 0; j < inputNum; ++j) { + if (pInput[j].num == 1) { + input[j] = pInput[j].data; + } else { + input[j] = pInput[j].data + i * pInput[j].bytes; + } + if (isNull(input[j], pInput[j].type)) { + hasNullInput = true; + break; } } + output = pOutput->data + i * pOutput->bytes; - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; + if (hasNullInput) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = floor(p[i]); - } + double base; + GET_TYPED_DATA(base, double, pInput[1].type, input[1]); + double v; + GET_TYPED_DATA(v, double, pInput[0].type, input[0]); + double result = pow(v, base); + SET_TYPED_DATA(output, pOutput->type, result); + } + + taosMemoryFree(input); + + return TSDB_CODE_SUCCESS; +} + +int32_t sqrtFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; } - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = sqrt(v); + SET_TYPED_DATA(output, pOutput->type, result); } + + return TSDB_CODE_SUCCESS; } -static void _tabs(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assignBasicParaInfo(pOutput, pLeft); - assert(numOfInput == 1); +int32_t sinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; } + output = pOutput->data + i * pOutput->bytes; - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; } - case TSDB_DATA_TYPE_TINYINT: { - int8_t* p = (int8_t*) pLeft->data; - int8_t* out = (int8_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = sin(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t cosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; } + output = pOutput->data + i * pOutput->bytes; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t* p = (int16_t*) pLeft->data; - int16_t* out = (int16_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; } - case TSDB_DATA_TYPE_INT: { - int32_t* p = (int32_t*) pLeft->data; - int32_t* out = (int32_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = cos(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; } + output = pOutput->data + i * pOutput->bytes; - case TSDB_DATA_TYPE_BIGINT: { - int64_t* p = (int64_t*) pLeft->data; - int64_t* out = (int64_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = tan(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t asinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; } - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = asin(v); + SET_TYPED_DATA(output, pOutput->type, result); } + + return TSDB_CODE_SUCCESS; } -static void tround(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assignBasicParaInfo(pOutput, pLeft); - assert(numOfInput == 1); +int32_t acosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = roundf(p[i]); + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = acos(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = atan(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + switch (pInput->type) { + case TSDB_DATA_TYPE_FLOAT: { + float v; + GET_TYPED_DATA(v, float, pInput->type, input); + float result = ceilf(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = ceil(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + default: { + memcpy(output, input, pInput->bytes); + break; } } + } - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = round(p[i]); + return TSDB_CODE_SUCCESS; +} + +int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + assignBasicParaInfo(pOutput, pInput); + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + switch (pInput->type) { + case TSDB_DATA_TYPE_FLOAT: { + float v; + GET_TYPED_DATA(v, float, pInput->type, input); + float result = floorf(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = floor(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; } + + default: { + memcpy(output, input, pInput->bytes); + break; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + assignBasicParaInfo(pOutput, pInput); + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; } - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + switch (pInput->type) { + case TSDB_DATA_TYPE_FLOAT: { + float v; + GET_TYPED_DATA(v, float, pInput->type, input); + float result = roundf(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = round(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + default: { + memcpy(output, input, pInput->bytes); + break; + } + } } + + return TSDB_CODE_SUCCESS; } static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { @@ -169,7 +560,7 @@ static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam } } - pOutput->data = realloc(pOutput->data, rowLen * num); + pOutput->data = taosMemoryRealloc(pOutput->data, rowLen * num); assert(pOutput->data); char* rstart = pOutput->data; @@ -284,13 +675,13 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa SScalarFuncParam rightOutput = {0}; if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) { - leftOutput.data = malloc(sizeof(int64_t) * numOfRows); + leftOutput.data = taosMemoryMalloc(sizeof(int64_t) * numOfRows); evaluateExprNodeTree(pLeft, numOfRows, &leftOutput, param, getSourceDataBlock); } // the right output has result from the right child syntax tree if (pRight->nodeType == TEXPR_BINARYEXPR_NODE || pRight->nodeType == TEXPR_UNARYEXPR_NODE) { - rightOutput.data = malloc(sizeof(int64_t) * numOfRows); + rightOutput.data = taosMemoryMalloc(sizeof(int64_t) * numOfRows); evaluateExprNodeTree(pRight, numOfRows, &rightOutput, param, getSourceDataBlock); } @@ -322,7 +713,7 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa void* outputBuf = pOutput->data; if (isStringOp(pExprs->_node.optr)) { - outputBuf = realloc(pOutput->data, (left.bytes + right.bytes) * left.num); + outputBuf = taosMemoryRealloc(pOutput->data, (left.bytes + right.bytes) * left.num); } OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC); @@ -345,7 +736,7 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa // reserve enough memory buffer if (isBinaryStringOp(pExprs->_node.optr)) { - void* outputBuf = realloc(pOutput->data, left.bytes * left.num); + void* outputBuf = taosMemoryRealloc(pOutput->data, left.bytes * left.num); assert(outputBuf != NULL); pOutput->data = outputBuf; } @@ -353,24 +744,24 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa OperatorFn(&left, pOutput); } - tfree(leftOutput.data); - tfree(rightOutput.data); + taosMemoryFreeClear(leftOutput.data); + taosMemoryFreeClear(rightOutput.data); return 0; } #endif -SScalarFunctionInfo scalarFunc[8] = { - {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, - {"floor", FUNCTION_TYPE_SCALAR, FUNCTION_FLOOR, tfloor}, - {"abs", FUNCTION_TYPE_SCALAR, FUNCTION_ABS, _tabs}, - {"round", FUNCTION_TYPE_SCALAR, FUNCTION_ROUND, tround}, - {"length", FUNCTION_TYPE_SCALAR, FUNCTION_LENGTH, tlength}, - {"concat", FUNCTION_TYPE_SCALAR, FUNCTION_CONCAT, tconcat}, - {"ltrim", FUNCTION_TYPE_SCALAR, FUNCTION_LTRIM, tltrim}, - {"rtrim", FUNCTION_TYPE_SCALAR, FUNCTION_RTRIM, trtrim}, -}; +//SScalarFunctionInfo scalarFunc[8] = { +// {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, +// {"floor", FUNCTION_TYPE_SCALAR, FUNCTION_FLOOR, tfloor}, +// {"abs", FUNCTION_TYPE_SCALAR, FUNCTION_ABS, _tabs}, +// {"round", FUNCTION_TYPE_SCALAR, FUNCTION_ROUND, tround}, +// {"length", FUNCTION_TYPE_SCALAR, FUNCTION_LENGTH, tlength}, +// {"concat", FUNCTION_TYPE_SCALAR, FUNCTION_CONCAT, tconcat}, +// {"ltrim", FUNCTION_TYPE_SCALAR, FUNCTION_LTRIM, tltrim}, +// {"rtrim", FUNCTION_TYPE_SCALAR, FUNCTION_RTRIM, trtrim}, +//}; void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; @@ -379,13 +770,13 @@ void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprI return; } - sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); + sas->colList = taosMemoryCalloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); for(int32_t i = 0; i < sas->numOfCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); sas->colList[i] = pColData->info; } - sas->data = calloc(sas->numOfCols, POINTER_BYTES); + sas->data = taosMemoryCalloc(sas->numOfCols, POINTER_BYTES); // set the input column data for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { @@ -395,7 +786,7 @@ void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprI } SScalarFunctionSupport* createScalarFuncSupport(int32_t num) { - SScalarFunctionSupport* pSupp = calloc(num, sizeof(SScalarFunctionSupport)); + SScalarFunctionSupport* pSupp = taosMemoryCalloc(num, sizeof(SScalarFunctionSupport)); return pSupp; } @@ -406,9 +797,9 @@ void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t n for(int32_t i = 0; i < num; ++i) { SScalarFunctionSupport* pSupp = &pSupport[i]; - tfree(pSupp->data); - tfree(pSupp->colList); + taosMemoryFreeClear(pSupp->data); + taosMemoryFreeClear(pSupp->colList); } - tfree(pSupport); -} \ No newline at end of file + taosMemoryFreeClear(pSupport); +} diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 17ac9b19fd5574b61beb25ec3198dace2992e5fe..2300f5a1d364bba6f0692a27a05df8d3fca8705e 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -305,7 +305,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t if (TSDB_DATA_TYPE_BINARY == inType) { if (varDataLen(pIn->data) >= bufSize) { bufSize = varDataLen(pIn->data) + 1; - tmp = realloc(tmp, bufSize); + tmp = taosMemoryRealloc(tmp, bufSize); } memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); @@ -313,13 +313,13 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t } else { if (varDataLen(pIn->data) * TSDB_NCHAR_SIZE >= bufSize) { bufSize = varDataLen(pIn->data) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); + tmp = taosMemoryRealloc(tmp, bufSize); } int len = taosUcs4ToMbs((TdUcs4*)varDataVal(pIn->data), varDataLen(pIn->data), tmp); if (len < 0){ sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); + taosMemoryFreeClear(tmp); return TSDB_CODE_QRY_APP_ERROR; } @@ -329,7 +329,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t (*func)(tmp, pOut, outType); } - tfree(tmp); + taosMemoryFreeClear(tmp); return TSDB_CODE_SUCCESS; } @@ -480,7 +480,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p paramOut1->bytes = param1->bytes; paramOut1->type = type; paramOut1->num = param1->num; - paramOut1->data = malloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); + paramOut1->data = taosMemoryMalloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); if (NULL == paramOut1->data) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -488,7 +488,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p code = vectorConvertImpl(param1, paramOut1); if (code) { - tfree(paramOut1->data); + taosMemoryFreeClear(paramOut1->data); return code; } } @@ -497,17 +497,17 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p paramOut2->bytes = param2->bytes; paramOut2->type = type; paramOut2->num = param2->num; - paramOut2->data = malloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); + paramOut2->data = taosMemoryMalloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); if (NULL == paramOut2->data) { - tfree(paramOut1->data); + taosMemoryFreeClear(paramOut1->data); return TSDB_CODE_QRY_OUT_OF_MEMORY; } paramOut2->orig.data = paramOut2->data; code = vectorConvertImpl(param2, paramOut2); if (code) { - tfree(paramOut1->data); - tfree(paramOut2->data); + taosMemoryFreeClear(paramOut1->data); + taosMemoryFreeClear(paramOut2->data); return code; } } @@ -523,7 +523,7 @@ void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, i SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -536,7 +536,7 @@ void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, i pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -681,7 +681,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -694,7 +694,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -774,7 +774,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -787,7 +787,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -863,7 +863,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -876,7 +876,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -953,7 +953,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -966,7 +966,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1040,7 +1040,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1053,7 +1053,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1206,7 +1206,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1219,7 +1219,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1303,7 +1303,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1316,7 +1316,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 328028a1d4675e408e77a16cad9fff36d4282d90..54f82eae2d2610013d682f2ea187e72f73b67fdc 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -71,7 +71,7 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { vnode->node.resType.type = dataType; if (IS_VAR_DATA_TYPE(dataType)) { - vnode->datum.p = (char *)malloc(varDataTLen(value)); + vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); vnode->node.resType.bytes = varDataLen(value); } else { @@ -102,7 +102,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; res->info.rows = rowNum; res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); @@ -113,7 +113,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); } @@ -122,7 +122,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); blockDataEnsureCapacity(res, rowNum); @@ -150,7 +150,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 1 + idx; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -325,7 +325,7 @@ TEST(columnTest, smallint_column_greater_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); blockDataDestroy(src); nodesDestroyNode(opNode); @@ -380,7 +380,7 @@ TEST(columnTest, int_column_greater_smallint_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -426,7 +426,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -491,7 +491,7 @@ TEST(columnTest, binary_column_in_binary_list) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -540,7 +540,7 @@ TEST(columnTest, binary_column_like_binary) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -588,7 +588,7 @@ TEST(columnTest, binary_column_is_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -635,7 +635,7 @@ TEST(columnTest, binary_column_is_not_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -674,7 +674,7 @@ TEST(opTest, smallint_column_greater_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -713,7 +713,7 @@ TEST(opTest, smallint_value_add_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -758,7 +758,7 @@ TEST(opTest, bigint_column_multi_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -801,7 +801,7 @@ TEST(opTest, smallint_column_and_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -839,7 +839,7 @@ TEST(opTest, smallint_column_or_float_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -879,7 +879,7 @@ TEST(opTest, smallint_column_or_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -924,7 +924,7 @@ TEST(opTest, binary_column_is_true) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -996,7 +996,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1065,7 +1065,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1135,7 +1135,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1204,7 +1204,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1277,7 +1277,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index d6a73d99bb1e0b46af55605ed1af14625a653304..f0025de6b878d6588317e3b90709dab7cb3c311d 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -39,6 +39,14 @@ #include "nodes.h" #include "tlog.h" +#define _DEBUG_PRINT_ 0 + +#if _DEBUG_PRINT_ +#define PRINTF(...) printf(__VA_ARGS__) +#else +#define PRINTF(...) +#endif + namespace { SColumnInfo createColumnInfo(int32_t colId, int32_t type, int32_t bytes) { @@ -71,7 +79,7 @@ void scltInitLogFile() { void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { if (newBlock) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 1; res->info.rows = rows; res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); @@ -106,7 +114,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { vnode->node.resType.type = dataType; if (IS_VAR_DATA_TYPE(dataType)) { - vnode->datum.p = (char *)malloc(varDataTLen(value)); + vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); vnode->node.resType.bytes = varDataTLen(value); } else { @@ -125,7 +133,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->dataBlockId = 0; if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; res->info.rows = rowNum; res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); @@ -136,7 +144,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); } @@ -145,7 +153,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); blockDataEnsureCapacity(res, rowNum); @@ -173,7 +181,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 1 + idx; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -1433,6 +1441,1645 @@ TEST(columnTest, greater_and_lower) { nodesDestroyNode(logicNode); } +void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t num, bool setVal) { + SScalarParam *input = (SScalarParam *)taosMemoryCalloc(1, sizeof(SScalarParam)); + int32_t bytes; + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + bytes = sizeof(int8_t); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + bytes = sizeof(int16_t); + break; + } + case TSDB_DATA_TYPE_INT: { + bytes = sizeof(int32_t); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + bytes = sizeof(int64_t); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + bytes = sizeof(float); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + bytes = sizeof(double); + break; + } + } + + input->type = type; + input->num = num; + input->data = taosMemoryCalloc(num, bytes); + input->bytes = bytes; + if (setVal) { + for (int32_t i = 0; i < num; ++i) { + memcpy(input->data + i * bytes, pVal, bytes); + } + } else { + memset(input->data, 0, num * bytes); + } + + *pInput = input; +} + +void scltDestroyDataBlock(SScalarParam *pInput) { + taosMemoryFree(pInput->data); + taosMemoryFree(pInput); +} + +TEST(ScalarFunctionTest, absFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), val_tinyint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_tinyint = -10; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), -val_tinyint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //SMALLINT + int16_t val_smallint = 10; + type = TSDB_DATA_TYPE_SMALLINT; + scltMakeDataBlock(&pInput, type, &val_smallint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int16_t *)pOutput->data + i), val_smallint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_smallint = -10; + scltMakeDataBlock(&pInput, type, &val_smallint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int16_t *)pOutput->data + i), -val_smallint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //INT + int32_t val_int = 10; + type = TSDB_DATA_TYPE_INT; + scltMakeDataBlock(&pInput, type, &val_int, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int32_t *)pOutput->data + i), val_int); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_int = -10; + scltMakeDataBlock(&pInput, type, &val_int, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int32_t *)pOutput->data + i), -val_int); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //BIGINT + int64_t val_bigint = 10; + type = TSDB_DATA_TYPE_BIGINT; + scltMakeDataBlock(&pInput, type, &val_bigint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)pOutput->data + i), val_bigint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_bigint = -10; + scltMakeDataBlock(&pInput, type, &val_bigint, rowNum, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)pOutput->data + i), -val_bigint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 10.15; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before ABS:%f\n", *(float *)pInput->data); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), val_float); + PRINTF("float after ABS:%f\n", *((float *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_float = -10.15; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before ABS:%f\n", *(float *)pInput->data); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), -val_float); + PRINTF("float after ABS:%f\n", *((float *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //DOUBLE + double val_double = 10.15; + type = TSDB_DATA_TYPE_DOUBLE; + scltMakeDataBlock(&pInput, type, &val_double, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), val_double); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_double = -10.15; + scltMakeDataBlock(&pInput, type, &val_double, rowNum, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), -val_double); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + +} + +TEST(ScalarFunctionTest, absFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 5; + int32_t type; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint + i; + PRINTF("tiny_int before ABS:%d\n", *((int8_t *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), val_tinyint + i); + PRINTF("tiny_int after ABS:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_tinyint = -10; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint + i; + PRINTF("tiny_int before ABS:%d\n", *((int8_t *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), -(val_tinyint + i)); + PRINTF("tiny_int after ABS:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //SMALLINT + int16_t val_smallint = 10; + type = TSDB_DATA_TYPE_SMALLINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int16_t *)pInput->data + i) = val_smallint + i; + PRINTF("small_int before ABS:%d\n", *((int16_t *)pInput->data + i)); + } + + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int16_t *)pOutput->data + i), val_smallint + i); + PRINTF("small_int after ABS:%d\n", *((int16_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_smallint = -10; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int16_t *)pInput->data + i) = val_smallint + i; + PRINTF("small_int before ABS:%d\n", *((int16_t *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int16_t *)pOutput->data + i), -(val_smallint + i)); + PRINTF("small_int after ABS:%d\n", *((int16_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //INT + int32_t val_int = 10; + type = TSDB_DATA_TYPE_INT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int32_t *)pInput->data + i) = val_int + i; + PRINTF("int before ABS:%d\n", *((int32_t *)pInput->data + i)); + } + + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int32_t *)pOutput->data + i), val_int + i); + PRINTF("int after ABS:%d\n", *((int32_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_int = -10; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int32_t *)pInput->data + i) = val_int + i; + PRINTF("int before ABS:%d\n", *((int32_t *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int32_t *)pOutput->data + i), -(val_int + i)); + PRINTF("int after ABS:%d\n", *((int32_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 10.15; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float + i; + PRINTF("float before ABS:%f\n", *((float *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), val_float + i); + PRINTF("float after ABS:%f\n", *((float *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_float = -10.15; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float + i; + PRINTF("float before ABS:%f\n", *((float *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), -(val_float + i)); + PRINTF("float after ABS:%f\n", *((float *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //DOUBLE + double val_double = 10.15; + type = TSDB_DATA_TYPE_DOUBLE; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((double *)pInput->data + i) = val_double + i; + PRINTF("double before ABS:%f\n", *((double *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), val_double + i); + PRINTF("double after ABS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_double = -10.15; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((double *)pInput->data + i) = val_double + i; + PRINTF("double before ABS:%f\n", *((double *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), -(val_double + i)); + PRINTF("double after ABS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, sinFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.42016703682664092; + + //TINYINT + int8_t val_tinyint = 13; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before SIN:%d\n", *((int8_t *)pInput->data)); + + code = sinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after SIN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 13.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before SIN:%f\n", *((float *)pInput->data)); + + code = sinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after SIN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + +} + +TEST(ScalarFunctionTest, sinFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {0.42016703682664092, 0.99060735569487035, 0.65028784015711683}; + + + //TINYINT + int8_t val_tinyint[] = {13, 14, 15}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before SIN:%d\n", *((int8_t *)pInput->data + i)); + } + + code = sinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after SIN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {13.00, 14.00, 15.00}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before SIN:%f\n", *((float *)pInput->data + i)); + } + + code = sinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after SIN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, cosFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.90744678145019619; + + //TINYINT + int8_t val_tinyint = 13; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before COS:%d\n", *((int8_t *)pInput->data)); + + code = cosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after COS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 13.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before COS:%f\n", *((float *)pInput->data)); + + code = cosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after COS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, cosFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {0.90744678145019619, 0.13673721820783361, -0.75968791285882131}; + + //TINYINT + int8_t val_tinyint[] = {13, 14, 15}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before COS:%d\n", *((int8_t *)pInput->data + i)); + } + + code = cosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after COS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {13.00, 14.00, 15.00}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before COS:%f\n", *((float *)pInput->data + i)); + } + + code = cosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after COS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, tanFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.46302113293648961; + + //TINYINT + int8_t val_tinyint = 13; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before TAN:%d\n", *((int8_t *)pInput->data)); + + code = tanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after TAN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 13.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before TAN:%f\n", *((float *)pInput->data)); + + code = tanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after TAN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, tanFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {0.46302113293648961, 7.24460661609480550, -0.85599340090851872}; + + //TINYINT + int8_t val_tinyint[] = {13, 14, 15}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before TAN:%d\n", *((int8_t *)pInput->data + i)); + } + + code = tanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after TAN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {13.00, 14.00, 15.00}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before TAN:%f\n", *((float *)pInput->data + i)); + } + + code = tanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after TAN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, asinFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 1.57079632679489656; + + //TINYINT + int8_t val_tinyint = 1; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before ASIN:%d\n", *((int8_t *)pInput->data)); + + code = asinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after ASIN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 1.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before ASIN:%f\n", *((float *)pInput->data)); + + code = asinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after ASIN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, asinFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {-1.57079632679489656, 0.0, 1.57079632679489656}; + + + //TINYINT + int8_t val_tinyint[] = {-1, 0, 1}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before ASIN:%d\n", *((int8_t *)pInput->data + i)); + } + + code = asinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after ASIN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-1.0, 0.0, 1.0}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before ASIN:%f\n", *((float *)pInput->data + i)); + } + + code = asinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after ASIN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, acosFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.0; + + //TINYINT + int8_t val_tinyint = 1; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before ACOS:%d\n", *((int8_t *)pInput->data)); + + code = acosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after ACOS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 1.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before ACOS:%f\n", *((float *)pInput->data)); + + code = acosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after ACOS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, acosFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {3.14159265358979312, 1.57079632679489656, 0.0}; + + //TINYINT + int8_t val_tinyint[] = {-1, 0, 1}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before ACOS:%d\n", *((int8_t *)pInput->data + i)); + } + + code = acosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after ACOS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-1.0, 0.0, 1.0}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before ACOS:%f\n", *((float *)pInput->data + i)); + } + + code = acosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after ACOS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, atanFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.78539816339744828; + + //TINYINT + int8_t val_tinyint = 1; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before ATAN:%d\n", *((int8_t *)pInput->data)); + + code = atanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after ATAN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 1.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before ATAN:%f\n", *((float *)pInput->data)); + + code = atanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after ATAN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, atanFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {-0.78539816339744828, 0.0, 0.78539816339744828}; + + //TINYINT + int8_t val_tinyint[] = {-1, 0, 1}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before ATAN:%d\n", *((int8_t *)pInput->data + i)); + } + + code = atanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after ATAN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-1.0, 0.0, 1.0}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before ATAN:%f\n", *((float *)pInput->data + i)); + } + + code = atanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after ATAN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, ceilFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result = 10.0; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("tiny_int before CEIL:%d\n", *((int8_t *)pInput->data)); + + code = ceilFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), (int8_t)result); + PRINTF("tiny_int after CEIL:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 9.5; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before CEIL:%f\n", *((float *)pInput->data)); + + code = ceilFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), (float)result); + PRINTF("float after CEIL:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, ceilFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result[] = {-10.0, 0.0, 10.0}; + + //TINYINT + int8_t val_tinyint[] = {-10, 0, 10}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before CEIL:%d\n", *((int8_t *)pInput->data + i)); + } + + code = ceilFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), result[i]); + PRINTF("tiny_int after CEIL:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-10.5, 0.0, 9.5}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before CEIL:%f\n", *((float *)pInput->data + i)); + } + + code = ceilFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), result[i]); + PRINTF("float after CEIL:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, floorFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result = 10.0; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("tiny_int before FLOOR:%d\n", *((int8_t *)pInput->data)); + + code = floorFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), (int8_t)result); + PRINTF("tiny_int after FLOOR:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 10.5; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before FLOOR:%f\n", *((float *)pInput->data)); + + code = floorFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), (float)result); + PRINTF("float after FLOOR:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, floorFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result[] = {-10.0, 0.0, 10.0}; + + //TINYINT + int8_t val_tinyint[] = {-10, 0, 10}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before FLOOR:%d\n", *((int8_t *)pInput->data + i)); + } + + code = floorFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), result[i]); + PRINTF("tiny_int after FLOOR:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-9.5, 0.0, 10.5}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before FLOOR:%f\n", *((float *)pInput->data + i)); + } + + code = floorFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), result[i]); + PRINTF("float after FLOOR:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, roundFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result = 10.0; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("tiny_int before ROUND:%d\n", *((int8_t *)pInput->data)); + + code = roundFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), (int8_t)result); + PRINTF("tiny_int after ROUND:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 9.5; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before ROUND:%f\n", *((float *)pInput->data)); + + code = roundFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), (float)result); + PRINTF("float after ROUND:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, roundFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result[] = {-10.0, 0.0, 10.0}; + + //TINYINT + int8_t val_tinyint[] = {-10, 0, 10}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before ROUND:%d\n", *((int8_t *)pInput->data + i)); + } + + code = roundFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), result[i]); + PRINTF("tiny_int after ROUND:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-9.5, 0.0, 9.5}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before ROUND:%f\n", *((float *)pInput->data + i)); + } + + code = roundFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), result[i]); + PRINTF("float after ROUND:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, sqrtFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 5.0; + + //TINYINT + int8_t val_tinyint = 25; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before SQRT:%d\n", *((int8_t *)pInput->data)); + + code = sqrtFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after SQRT:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 25.0; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before SQRT:%f\n", *((float *)pInput->data)); + + code = sqrtFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after SQRT:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, sqrtFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {5.0, 9.0, 10.0}; + + //TINYINT + int8_t val_tinyint[] = {25, 81, 100}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before SQRT:%d\n", *((int8_t *)pInput->data + i)); + } + + code = sqrtFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after SQRT:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {25.0, 81.0, 100.0}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before SQRT:%f\n", *((float *)pInput->data + i)); + } + + code = sqrtFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after SQRT:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, logFunction_constant) { + SScalarParam *pInput, *pOutput; + SScalarParam *input[2]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 3.0; + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + + //TINYINT + int8_t val_tinyint[] = {27, 3}; + type = TSDB_DATA_TYPE_TINYINT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, &val_tinyint[i], 1, true); + pInput[i] = *input[i]; + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before LOG: %d,%d\n", *((int8_t *)pInput[0].data), + *((int8_t *)pInput[1].data)); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after LOG:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {64.0, 4.0}; + type = TSDB_DATA_TYPE_FLOAT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, &val_float[i], 1, true); + pInput[i] = *input[i]; + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before LOG: %f,%f\n", *((float *)pInput[0].data), + *((float *)pInput[1].data)); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after LOG:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //TINYINT AND FLOAT + int8_t param0 = 64; + float param1 = 4.0; + scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, ¶m0, 1, true); + pInput[0] = *input[0]; + scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, ¶m1, 1, true); + pInput[1] = *input[1]; + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + PRINTF("tiny_int,float before LOG: %d,%f\n", *((int8_t *)pInput[0].data), *((float *)pInput[1].data)); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int,float after LOG:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + taosMemoryFree(pInput); +} + +TEST(ScalarFunctionTest, logFunction_column) { + SScalarParam *pInput, *pOutput; + SScalarParam *input[2]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {2.0, 4.0, 3.0}; + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + + //TINYINT + int8_t val_tinyint[2][3] = {{25, 81, 64}, {5, 3, 4}}; + type = TSDB_DATA_TYPE_TINYINT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, 0, rowNum, false); + pInput[i] = *input[i]; + for (int32_t j = 0; j < rowNum; ++j) { + *((int8_t *)pInput[i].data + j) = val_tinyint[i][j]; + } + PRINTF("tiny_int before LOG:%d,%d,%d\n", *((int8_t *)pInput[i].data + 0), + *((int8_t *)pInput[i].data + 1), + *((int8_t *)pInput[i].data + 2)); + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after LOG:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[2][3] = {{25.0, 81.0, 64.0}, {5.0, 3.0, 4.0}}; + type = TSDB_DATA_TYPE_FLOAT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, 0, rowNum, false); + pInput[i] = *input[i]; + for (int32_t j = 0; j < rowNum; ++j) { + *((float *)pInput[i].data + j) = val_float[i][j]; + } + PRINTF("float before LOG:%f,%f,%f\n", *((float *)pInput[i].data + 0), + *((float *)pInput[i].data + 1), + *((float *)pInput[i].data + 2)); + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after LOG:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //TINYINT AND FLOAT + int8_t param0[] = {25, 81, 64}; + float param1[] = {5.0, 3.0, 4.0}; + scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, 0, rowNum, false); + pInput[0] = *input[0]; + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput[0].data + i) = param0[i]; + } + scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, 0, rowNum, false); + pInput[1] = *input[1]; + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput[1].data + i) = param1[i]; + } + PRINTF("tiny_int, float before LOG:{%d,%f}, {%d,%f}, {%d,%f}\n", *((int8_t *)pInput[0].data + 0), *((float *)pInput[1].data + 0), + *((int8_t *)pInput[0].data + 1), *((float *)pInput[1].data + 1), + *((int8_t *)pInput[0].data + 2), *((float *)pInput[1].data + 2)); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int,float after LOG:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + taosMemoryFree(pInput); +} + +TEST(ScalarFunctionTest, powFunction_constant) { + SScalarParam *pInput, *pOutput; + SScalarParam *input[2]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 16.0; + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + + //TINYINT + int8_t val_tinyint[] = {2, 4}; + type = TSDB_DATA_TYPE_TINYINT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, &val_tinyint[i], 1, true); + pInput[i] = *input[i]; + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before POW: %d,%d\n", *((int8_t *)pInput[0].data), + *((int8_t *)pInput[1].data)); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after POW:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {2.0, 4.0}; + type = TSDB_DATA_TYPE_FLOAT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, &val_float[i], 1, true); + pInput[i] = *input[i]; + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before POW: %f,%f\n", *((float *)pInput[0].data), + *((float *)pInput[1].data)); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after POW:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //TINYINT AND FLOAT + int8_t param0 = 2; + float param1 = 4.0; + scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, ¶m0, 1, true); + pInput[0] = *input[0]; + scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, ¶m1, 1, true); + pInput[1] = *input[1]; + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + PRINTF("tiny_int,float before POW: %d,%f\n", *((int8_t *)pInput[0].data), *((float *)pInput[1].data)); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int,float after POW:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + taosMemoryFree(pInput); +} + +TEST(ScalarFunctionTest, powFunction_column) { + SScalarParam *pInput, *pOutput; + SScalarParam *input[2]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {32.0, 27.0, 16.0}; + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + + //TINYINT + int8_t val_tinyint[2][3] = {{2, 3, 4}, {5, 3, 2}}; + type = TSDB_DATA_TYPE_TINYINT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, 0, rowNum, false); + pInput[i] = *input[i]; + for (int32_t j = 0; j < rowNum; ++j) { + *((int8_t *)pInput[i].data + j) = val_tinyint[i][j]; + } + PRINTF("tiny_int before POW:%d,%d,%d\n", *((int8_t *)pInput[i].data + 0), + *((int8_t *)pInput[i].data + 1), + *((int8_t *)pInput[i].data + 2)); + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after POW:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[2][3] = {{2.0, 3.0, 4.0}, {5.0, 3.0, 2.0}}; + type = TSDB_DATA_TYPE_FLOAT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, 0, rowNum, false); + pInput[i] = *input[i]; + for (int32_t j = 0; j < rowNum; ++j) { + *((float *)pInput[i].data + j) = val_float[i][j]; + } + PRINTF("float before POW:%f,%f,%f\n", *((float *)pInput[i].data + 0), + *((float *)pInput[i].data + 1), + *((float *)pInput[i].data + 2)); + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after POW:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //TINYINT AND FLOAT + int8_t param0[] = {2, 3, 4}; + float param1[] = {5.0, 3.0, 2.0}; + scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, 0, rowNum, false); + pInput[0] = *input[0]; + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput[0].data + i) = param0[i]; + } + scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, 0, rowNum, false); + pInput[1] = *input[1]; + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput[1].data + i) = param1[i]; + } + PRINTF("tiny_int, float before POW:{%d,%f}, {%d,%f}, {%d,%f}\n", *((int8_t *)pInput[0].data + 0), *((float *)pInput[1].data + 0), + *((int8_t *)pInput[0].data + 1), *((float *)pInput[1].data + 1), + *((int8_t *)pInput[0].data + 2), *((float *)pInput[1].data + 2)); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int,float after POW:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + taosMemoryFree(pInput); +} int main(int argc, char** argv) { taosSeedRand(taosGetTimestampSec()); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 2776059f66673696a001b66a5edd86065959f0f1..518da6e2b8bbf7d21706034e4f28621105bcc7ff 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -25,11 +25,11 @@ extern "C" { #include "planner.h" #include "scheduler.h" #include "thash.h" +#include "trpc.h" #define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 -#define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 20 // unit is TSDB_TABLE_NUM_UNIT - +#define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT #define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA @@ -45,7 +45,7 @@ typedef struct SSchTrans { typedef struct SSchHbTrans { SRWLatch lock; - uint64_t seqId; + SRpcCtx rpcCtx; SSchTrans trans; } SSchHbTrans; @@ -77,12 +77,23 @@ typedef struct SSchedulerMgmt { SHashObj *hbConnections; } SSchedulerMgmt; -typedef struct SSchCallbackParam { - uint64_t queryId; - int64_t refId; - uint64_t taskId; - void *transport; -} SSchCallbackParam; +typedef struct SSchCallbackParamHeader { + bool isHbParam; +} SSchCallbackParamHeader; + +typedef struct SSchTaskCallbackParam { + SSchCallbackParamHeader head; + uint64_t queryId; + int64_t refId; + uint64_t taskId; + void *transport; +} SSchTaskCallbackParam; + +typedef struct SSchHbCallbackParam { + SSchCallbackParamHeader head; + SQueryNodeEpId nodeEpId; + void *transport; +} SSchHbCallbackParam; typedef struct SSchFlowControl { SRWLatch lock; @@ -92,6 +103,11 @@ typedef struct SSchFlowControl { SArray *taskList; // Element is SSchTask* } SSchFlowControl; +typedef struct SSchNodeInfo { + SQueryNodeAddr addr; + void *handle; +} SSchNodeInfo; + typedef struct SSchLevel { int32_t level; int8_t status; @@ -113,10 +129,11 @@ typedef struct SSchTask { int32_t msgLen; // msg length int8_t status; // task status int32_t lastMsgType; // last sent msg type + int32_t tryTimes; // task already tried times SQueryNodeAddr succeedAddr; // task executed success node address int8_t candidateIdx; // current try condidation index SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr - SArray *execAddrs; // all tried node for current task, element is SQueryNodeAddr + SArray *execNodes; // all tried node for current task, element is SSchNodeInfo SQueryProfileSummary summary; // task execution summary int32_t childReady; // child task ready number SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* @@ -136,6 +153,7 @@ typedef struct SSchJob { uint64_t queryId; SSchJobAttr attr; int32_t levelNum; + int32_t taskNum; void *transport; SArray *nodeList; // qnode/vnode list, element is SQueryNodeAddr SArray *levels; // Element is SQueryLevel, starting from 0. SArray @@ -154,7 +172,8 @@ typedef struct SSchJob { int32_t remoteFetch; SSchTask *fetchTask; int32_t errCode; - void *res; //TODO free it or not + SArray *errList; // SArray + void *resData; //TODO free it or not int32_t resNumOfRows; const char *sql; SQueryProfileSummary summary; @@ -168,27 +187,35 @@ extern SSchedulerMgmt schMgmt; #define SCH_SET_TASK_LASTMSG_TYPE(_task, _type) do { if(_task) { atomic_store_32(&(_task)->lastMsgType, _type); } } while (0) #define SCH_GET_TASK_LASTMSG_TYPE(_task) ((_task) ? atomic_load_32(&(_task)->lastMsgType) : -1) -#define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) -#define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY) -#define SCH_TASK_NO_NEED_DROP(task) ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY) +#define SCH_IS_DATA_SRC_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) +#define SCH_IS_DATA_SRC_TASK(task) (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) +#define SCH_IS_LEAF_TASK(_job, _task) (((_task)->level->level + 1) == (_job)->levelNum) #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) +#define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task)) + +#define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL) +#define SCH_SET_TASK_HANDLE(_task, _handle) ((_task)->handle = (_handle)) #define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st) #define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) +#define SCH_GET_JOB_STATUS_STR(job) jobTaskStatusStr(SCH_GET_JOB_STATUS(job)) #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true #define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) -#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEAF_TASK(_job, _task) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) +#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEAF_TASK(_job, _task) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) #define SCH_SET_JOB_TYPE(_job, type) (_job)->attr.queryJob = ((type) != SUBPLAN_TYPE_MODIFY) #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) #define SCH_JOB_NEED_FETCH(_job) SCH_IS_QUERY_JOB(_job) -#define SCH_IS_LEAF_TASK(_job, _task) (((_task)->level->level + 1) == (_job)->levelNum) +#define SCH_IS_WAIT_ALL_JOB(_job) (!SCH_IS_QUERY_JOB(_job)) +#define SCH_IS_NEED_DROP_JOB(_job) (SCH_IS_QUERY_JOB(_job)) + #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) #define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) #define SCH_SWITCH_EPSET(_addr) ((_addr)->epSet.inUse = ((_addr)->epSet.inUse + 1) % (_addr)->epSet.numOfEps) +#define SCH_TASK_NUM_OF_EPS(_addr) ((_addr)->epSet.numOfEps) #define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) @@ -197,6 +224,8 @@ extern SSchedulerMgmt schMgmt; qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__) +#define SCH_TASK_DLOGL(param, ...) \ + qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__) @@ -220,6 +249,8 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); int32_t schFetchFromRemote(SSchJob *pJob); int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); +int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId); +int32_t schCloneSMsgSendInfo(void *src, void **dst); #ifdef __cplusplus diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 6b1ca25d932d82274c3543faf02d3039250b8b3a..5af13d97ca0e249f25618abd9950bd26d721fa2e 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -18,6 +18,7 @@ #include "schedulerInt.h" #include "tmsg.h" #include "tref.h" +#include "trpc.h" SSchedulerMgmt schMgmt = {0}; @@ -56,21 +57,41 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->level = pLevel; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); pTask->taskId = schGenTaskId(); - pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); - if (NULL == pTask->execAddrs) { - SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CANDIDATE_EP_NUM); + pTask->execNodes = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SSchNodeInfo)); + if (NULL == pTask->execNodes) { + SCH_TASK_ELOG("taosArrayInit %d execNodes failed", SCH_MAX_CANDIDATE_EP_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; } -void schFreeTask(SSchTask *pTask) { +void schFreeRpcCtx(SRpcCtx *pCtx) { + if (NULL == pCtx) { + return; + } + void *pIter = taosHashIterate(pCtx->args, NULL); + while (pIter) { + SRpcCtxVal *ctxVal = (SRpcCtxVal *)pIter; + + (*ctxVal->freeFunc)(ctxVal->val); + + pIter = taosHashIterate(pCtx->args, pIter); + } + + taosHashCleanup(pCtx->args); + + if (pCtx->brokenVal.freeFunc) { + (*pCtx->brokenVal.freeFunc)(pCtx->brokenVal.val); + } +} + +void schFreeTask(SSchTask* pTask) { if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } - tfree(pTask->msg); + taosMemoryFreeClear(pTask->msg); if (pTask->children) { taosArrayDestroy(pTask->children); @@ -80,8 +101,8 @@ void schFreeTask(SSchTask *pTask) { taosArrayDestroy(pTask->parents); } - if (pTask->execAddrs) { - taosArrayDestroy(pTask->execAddrs); + if (pTask->execNodes) { + taosArrayDestroy(pTask->execNodes); } } @@ -98,34 +119,67 @@ static FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t lastMsgType = SCH_GET_TASK_LASTMSG_TYPE(pTask); - + int32_t taskStatus = SCH_GET_TASK_STATUS(pTask); + int32_t reqMsgType = msgType - 1; switch (msgType) { - case TDMT_VND_CREATE_TABLE_RSP: - case TDMT_VND_SUBMIT_RSP: - case TDMT_VND_QUERY_RSP: + case TDMT_SCH_LINK_BROKEN: + return TSDB_CODE_SUCCESS; + case TDMT_VND_QUERY_RSP: // query_rsp may be processed later than ready_rsp + if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) { + SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + } + + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + } + + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + return TSDB_CODE_SUCCESS; case TDMT_VND_RES_READY_RSP: - case TDMT_VND_FETCH_RSP: - case TDMT_VND_DROP_TASK: - if (lastMsgType != (msgType - 1)) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), - TMSG_INFO(msgType)); + reqMsgType = TDMT_VND_QUERY; + if (lastMsgType != reqMsgType && -1 != lastMsgType) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", (lastMsgType > 0 ? TMSG_INFO(lastMsgType) : "null"), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - - if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && - SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%s", SCH_GET_TASK_STATUS(pTask), - TMSG_INFO(msgType)); + + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + return TSDB_CODE_SUCCESS; + case TDMT_VND_FETCH_RSP: + if (lastMsgType != reqMsgType && -1 != lastMsgType) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + return TSDB_CODE_SUCCESS; + case TDMT_VND_CREATE_TABLE_RSP: + case TDMT_VND_SUBMIT_RSP: break; default: - SCH_TASK_ELOG("unknown rsp msg, type:%s, status:%d", TMSG_INFO(msgType), SCH_GET_TASK_STATUS(pTask)); - + SCH_TASK_ELOG("unknown rsp msg, type:%s, status:%s", TMSG_INFO(msgType), jobTaskStatusStr(taskStatus)); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + if (lastMsgType != reqMsgType) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; @@ -185,7 +239,7 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; default: - SCH_JOB_ELOG("invalid job status:%d", oriStatus); + SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -193,7 +247,7 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { continue; } - SCH_JOB_DLOG("job status updated from %d to %d", oriStatus, newStatus); + SCH_JOB_DLOG("job status updated from %s to %s", jobTaskStatusStr(oriStatus), jobTaskStatusStr(newStatus)); break; } @@ -202,8 +256,7 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { _return: - SCH_JOB_ELOG("invalid job status update, from %d to %d", oriStatus, newStatus); - + SCH_JOB_ELOG("invalid job status update, from %s to %s", jobTaskStatusStr(oriStatus), jobTaskStatusStr(newStatus)); SCH_ERR_RET(code); } @@ -302,12 +355,16 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr) { - if (NULL == taosArrayPush(pTask->execAddrs, addr)) { - SCH_TASK_ELOG("taosArrayPush addr to execAddr list failed, errno:%d", errno); +int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, void *handle) { + SSchNodeInfo nodeInfo = {.addr = *addr, .handle = handle}; + + if (NULL == taosArrayPush(pTask->execNodes, &nodeInfo)) { + SCH_TASK_ELOG("taosArrayPush nodeInfo to execNodes list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + SCH_TASK_DLOG("task execNode recorded, handle:%p", handle); + return TSDB_CODE_SUCCESS; } @@ -402,6 +459,8 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + ++pJob->taskNum; } SCH_JOB_DLOG("level initialized, taskNum:%d", taskNum); @@ -459,7 +518,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (addNum <= 0) { SCH_TASK_ELOG("no available execNode as candidates, nodeNum:%d", nodeNum); - return TSDB_CODE_QRY_INVALID_INPUT; + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } /* @@ -493,7 +552,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { - SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); } else { SCH_TASK_DLOG("task removed from execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); } @@ -502,8 +561,7 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - - SCH_TASK_ELOG("task already in succTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_ELOG("task already in succTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -522,15 +580,15 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = false; if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { - SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); } int32_t code = taosHashPut(pJob->failTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - - SCH_TASK_WLOG("task already in failTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); + + SCH_TASK_WLOG("task already in failTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -547,15 +605,15 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) { - SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); } int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - - SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); + + SCH_TASK_ELOG("task already in execTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -571,23 +629,48 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { - // TODO set retry or not based on task type/errCode/retry times/job status/available eps... - - *needRetry = false; + int8_t status = 0; + ++pTask->tryTimes; + + if (schJobNeedToStop(pJob, &status)) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry cause of job status, job status:%s", jobTaskStatusStr(status)); + return TSDB_CODE_SUCCESS; + } - return TSDB_CODE_SUCCESS; + if (pTask->tryTimes >= REQUEST_MAX_TRY_TIMES) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry since reach max try times, tryTimes:%d", pTask->tryTimes); + return TSDB_CODE_SUCCESS; + } + + if (!NEED_SCHEDULER_RETRY_ERROR(errCode)) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode)); + return TSDB_CODE_SUCCESS; + } // TODO CHECK epList/condidateList if (SCH_IS_DATA_SRC_TASK(pTask)) { + if (pTask->tryTimes >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry since all ep tried, tryTimes:%d, epNum:%d", pTask->tryTimes, SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); + return TSDB_CODE_SUCCESS; + } } else { int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); if ((pTask->candidateIdx + 1) >= candidateNum) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", pTask->candidateIdx, candidateNum); return TSDB_CODE_SUCCESS; } - - ++pTask->candidateIdx; } + + *needRetry = true; + SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->tryTimes, errCode, tstrerror(errCode)); + + return TSDB_CODE_SUCCESS; } int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { @@ -609,64 +692,62 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { +int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { int32_t code = 0; SSchHbTrans *hb = NULL; - while (true) { - hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); - if (NULL == hb) { - code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), trans, sizeof(SSchHbTrans)); - if (code) { - if (HASH_NODE_EXIST(code)) { - continue; - } + hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } - qError("taosHashPut hb trans failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(code); - } + SCH_LOCK(SCH_WRITE, &hb->lock); + memcpy(&hb->trans, trans, sizeof(*trans)); + SCH_UNLOCK(SCH_WRITE, &hb->lock); - qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 - ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", - trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, - trans->trans.transHandle); + qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, handle:%p", + schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->transInst, + trans->transHandle); - return TSDB_CODE_SUCCESS; - } + return TSDB_CODE_SUCCESS; +} - break; +void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { + if (TSDB_CODE_SUCCESS == errCode) { + return; } - SCH_LOCK(SCH_WRITE, &hb->lock); - - if (hb->seqId >= trans->seqId) { - qDebug("hb trans seqId is old, seqId:%" PRId64 ", currentId:%" PRId64 ", nodeId:%d, fqdn:%s, port:%d", trans->seqId, - hb->seqId, epId->nodeId, epId->ep.fqdn, epId->ep.port); + int32_t origCode = atomic_load_32(&pJob->errCode); + if (TSDB_CODE_SUCCESS == origCode) { + if (origCode == atomic_val_compare_exchange_32(&pJob->errCode, origCode, errCode)) { + goto _return; + } - SCH_UNLOCK(SCH_WRITE, &hb->lock); - return TSDB_CODE_SUCCESS; + origCode = atomic_load_32(&pJob->errCode); } - hb->seqId = trans->seqId; - memcpy(&hb->trans, &trans->trans, sizeof(trans->trans)); - - SCH_UNLOCK(SCH_WRITE, &hb->lock); + if (NEED_CLIENT_HANDLE_ERROR(origCode)) { + return; + } + + if (NEED_CLIENT_HANDLE_ERROR(errCode)) { + atomic_store_32(&pJob->errCode, errCode); + goto _return; + } - qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 - ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", - trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, - trans->trans.transHandle); + return; + +_return: - return TSDB_CODE_SUCCESS; + SCH_JOB_DLOG("job errCode updated to %x - %s", errCode, tstrerror(errCode)); } int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCode) { // if already FAILED, no more processing SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, status)); - if (errCode) { - atomic_store_32(&pJob->errCode, errCode); - } + schUpdateJobErrCode(pJob, errCode); if (atomic_load_8(&pJob->userFetch) || pJob->attr.syncSchedule) { tsem_post(&pJob->rspSem); @@ -720,8 +801,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { - SCH_TASK_DLOG("task failed not processed cause of job status, job status:%d", status); - + SCH_TASK_DLOG("task failed not processed cause of job status, job status:%s", jobTaskStatusStr(status)); SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -740,23 +820,23 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) if (SCH_GET_TASK_STATUS(pTask) == JOB_TASK_STATUS_EXECUTING) { SCH_ERR_JRET(schMoveTaskToFailList(pJob, pTask, &moved)); } else { - SCH_TASK_DLOG("task already done, no more failure process, status:%d", SCH_GET_TASK_STATUS(pTask)); - return TSDB_CODE_SUCCESS; + SCH_TASK_ELOG("task not in executing list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED); - - if (SCH_TASK_NEED_WAIT_ALL(pTask)) { + + if (SCH_IS_WAIT_ALL_JOB(pJob)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskFailed++; taskDone = pTask->level->taskSucceed + pTask->level->taskFailed; SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); - atomic_store_32(&pJob->errCode, errCode); - + schUpdateJobErrCode(pJob, errCode); + if (taskDone < pTask->level->taskNum) { - SCH_TASK_DLOG("not all tasks done, done:%d, all:%d", taskDone, pTask->level->taskNum); - SCH_ERR_RET(errCode); + SCH_TASK_DLOG("need to wait other tasks, doneNum:%d, allNum:%d", taskDone, pTask->level->taskNum); + SCH_RET(errCode); } } } else { @@ -775,7 +855,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { bool moved = false; int32_t code = 0; - SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_DLOG("taskOnSuccess, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_JRET(schMoveTaskToSuccList(pJob, pTask, &moved)); @@ -787,9 +867,8 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { - int32_t taskDone = 0; - - if (SCH_TASK_NEED_WAIT_ALL(pTask)) { + int32_t taskDone = 0; + if (SCH_IS_WAIT_ALL_JOB(pJob)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskSucceed++; taskDone = pTask->level->taskSucceed + pTask->level->taskFailed; @@ -860,11 +939,11 @@ int32_t schFetchFromRemote(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } - void *res = atomic_load_ptr(&pJob->res); - if (res) { + void *resData = atomic_load_ptr(&pJob->resData); + if (resData) { atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); - SCH_JOB_DLOG("res already fetched, res:%p", res); + SCH_JOB_DLOG("res already fetched, res:%p", resData); return TSDB_CODE_SUCCESS; } @@ -886,8 +965,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { - SCH_TASK_ELOG("rsp not processed cause of job status, job status:%d", status); - + SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), rspCode); SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -895,45 +973,59 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch switch (msgType) { case TDMT_VND_CREATE_TABLE_RSP: { + SVCreateTbBatchRsp batchRsp = {0}; + if (msg) { + tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp); + if (batchRsp.rspList) { + int32_t num = taosArrayGetSize(batchRsp.rspList); + for (int32_t i = 0; i < num; ++i) { + SVCreateTbRsp *rsp = taosArrayGet(batchRsp.rspList, i); + if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { + taosArrayDestroy(batchRsp.rspList); + SCH_ERR_JRET(rsp->code); + } + } + + taosArrayDestroy(batchRsp.rspList); + } + } + SCH_ERR_JRET(rspCode); SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - break; } case TDMT_VND_SUBMIT_RSP: { -#if 0 // TODO OPEN THIS - SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; - - if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); - } - - pJob->resNumOfRows += rsp->affectedRows; -#else + if (msg) { + SSubmitRsp *rsp = (SSubmitRsp *)msg; + SCH_ERR_JRET(rsp->code); + } SCH_ERR_JRET(rspCode); SSubmitRsp *rsp = (SSubmitRsp *)msg; if (rsp) { pJob->resNumOfRows += rsp->affectedRows; } -#endif SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); break; } case TDMT_VND_QUERY_RSP: { - SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - - SCH_ERR_JRET(rspCode); - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - SCH_ERR_JRET(rsp->code); - - SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); - - break; + SQueryTableRsp rsp = {0}; + if (msg) { + tDeserializeSQueryTableRsp(msg, msgSize, &rsp); + SCH_ERR_JRET(rsp.code); + } + + SCH_ERR_JRET(rspCode); + + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + //SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); + + break; } case TDMT_VND_RES_READY_RSP: { SResReadyRsp *rsp = (SResReadyRsp *)msg; @@ -943,7 +1035,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } SCH_ERR_JRET(rsp->code); - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); break; @@ -956,13 +1047,13 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - if (pJob->res) { - SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->res); - tfree(rsp); + if (pJob->resData) { + SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->resData); + taosMemoryFreeClear(rsp); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } - atomic_store_ptr(&pJob->res, rsp); + atomic_store_ptr(&pJob->resData, rsp); atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); if (rsp->completed) { @@ -980,8 +1071,12 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); break; } + case TDMT_SCH_LINK_BROKEN: + SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode)); + SCH_ERR_JRET(rspCode); + break; default: - SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%s", msgType, SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -994,12 +1089,12 @@ _return: int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0; - SSchCallbackParam *pParam = (SSchCallbackParam *)param; + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; SSchTask *pTask = NULL; SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { - qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } @@ -1017,9 +1112,9 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in } pTask = *task; - SCH_TASK_DLOG("rsp msg received, type:%s, code:%s", TMSG_INFO(msgType), tstrerror(rspCode)); + SCH_TASK_DLOG("rsp msg received, type:%s, handle:%p, code:%s", TMSG_INFO(msgType), pMsg->handle, tstrerror(rspCode)); - pTask->handle = pMsg->handle; + SCH_SET_TASK_HANDLE(pTask, pMsg->handle); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: @@ -1028,7 +1123,7 @@ _return: schReleaseJob(pParam->refId); } - tfree(param); + taosMemoryFreeClear(param); SCH_RET(code); } @@ -1053,7 +1148,7 @@ int32_t schHandleReadyCallback(void *param, const SDataBuf *pMsg, int32_t code) } int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { - SSchCallbackParam *pParam = (SSchCallbackParam *)param; + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); } @@ -1064,24 +1159,22 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { } SSchedulerHbRsp rsp = {0}; - - SSchCallbackParam *pParam = (SSchCallbackParam *)param; - if (tDeserializeSSchedulerHbRsp(pMsg->pData, pMsg->len, &rsp)) { qError("invalid hb rsp msg, size:%d", pMsg->len); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - if (rsp.seqId != (uint64_t)-1) { - SSchHbTrans trans = {0}; - trans.seqId = rsp.seqId; - trans.trans.transInst = pParam->transport; - trans.trans.transHandle = pMsg->handle; + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SCH_RET(schUpdateHbConnection(&rsp.epId, &trans)); - } + SSchTrans trans = {0}; + trans.transInst = pParam->transport; + trans.transHandle = pMsg->handle; + + SCH_ERR_RET(schUpdateHbConnection(&rsp.epId, &trans)); int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); + qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, rsp.epId.ep.port); + for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i); @@ -1094,6 +1187,8 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { } // TODO + + SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, jobTaskStatusStr(taskStatus->status)); schReleaseJob(taskStatus->refId); } @@ -1105,6 +1200,26 @@ _return: SCH_RET(code); } +int32_t schHandleLinkBrokenCallback(void *param, const SDataBuf *pMsg, int32_t code) { + SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param; + rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + + qDebug("handle %p is broken", pMsg->handle); + + if (head->isHbParam) { + SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param; + SSchTrans trans = {.transInst = hbParam->transport, .transHandle = NULL}; + SCH_ERR_RET(schUpdateHbConnection(&hbParam->nodeEpId, &trans)); + + SCH_ERR_RET(schBuildAndSendHbMsg(&hbParam->nodeEpId)); + } else { + SCH_ERR_RET(schHandleCallback(param, pMsg, TDMT_SCH_LINK_BROKEN, code)); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { switch (msgType) { case TDMT_VND_CREATE_TABLE: @@ -1128,6 +1243,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_VND_QUERY_HEARTBEAT: *fp = schHandleHbCallback; break; + case TDMT_SCH_LINK_BROKEN: + *fp = schHandleLinkBrokenCallback; + break; default: qError("unknown msg type for callback, msgType:%d", msgType); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -1136,21 +1254,335 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { return TSDB_CODE_SUCCESS; } -int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet *epSet, int32_t msgType, void *msg, - uint32_t msgSize) { +void schFreeRpcCtxVal(const void *arg) { + if (NULL == arg) { + return; + } + + SMsgSendInfo* pMsgSendInfo = (SMsgSendInfo *)arg; + taosMemoryFreeClear(pMsgSendInfo->param); + taosMemoryFreeClear(pMsgSendInfo); +} + +int32_t schMakeTaskCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { + SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param->queryId = pJob->queryId; + param->refId = pJob->refId; + param->taskId = SCH_TASK_ID(pTask); + param->transport = pJob->transport; + + *pParam = param; + + return TSDB_CODE_SUCCESS; +} + +int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { + SSchHbCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param->head.isHbParam = true; + + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + + param->nodeEpId.nodeId = addr->nodeId; + memcpy(¶m->nodeEpId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + param->transport = pJob->transport; + + *pParam = param; + + return TSDB_CODE_SUCCESS; +} + + +int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb) { + int32_t code = 0; + SMsgSendInfo* pMsgSendInfo = NULL; + + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (isHb) { + SCH_ERR_JRET(schMakeHbCallbackParam(pJob, pTask, &pMsgSendInfo->param)); + } else { + SCH_ERR_JRET(schMakeTaskCallbackParam(pJob, pTask, &pMsgSendInfo->param)); + } + + int32_t msgType = TDMT_SCH_LINK_BROKEN; + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); + + pMsgSendInfo->fp = fp; + + brokenVal->msgType = msgType; + brokenVal->val = pMsgSendInfo; + brokenVal->clone = schCloneSMsgSendInfo; + brokenVal->freeFunc = schFreeRpcCtxVal; + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(pMsgSendInfo->param); + taosMemoryFreeClear(pMsgSendInfo); + + SCH_RET(code); +} + +int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { + int32_t code = 0; + SSchTaskCallbackParam *param = NULL; + SMsgSendInfo* pMsgSendInfo = NULL; + + pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (NULL == pCtx->args) { + SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t msgType = TDMT_VND_RES_READY_RSP; + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_RES_READY, &fp)); + + param->queryId = pJob->queryId; + param->refId = pJob->refId; + param->taskId = SCH_TASK_ID(pTask); + param->transport = pJob->transport; + + pMsgSendInfo->param = param; + pMsgSendInfo->fp = fp; + + SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo, .freeFunc = schFreeRpcCtxVal}; + if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { + SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, false)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosHashCleanup(pCtx->args); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); + + SCH_RET(code); +} + +int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { + int32_t code = 0; + SSchHbCallbackParam *param = NULL; + SMsgSendInfo* pMsgSendInfo = NULL; + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SQueryNodeEpId epId = {0}; + + epId.nodeId = addr->nodeId; + memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + + pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (NULL == pCtx->args) { + SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t msgType = TDMT_VND_QUERY_HEARTBEAT_RSP; + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_QUERY_HEARTBEAT, &fp)); + + param->nodeEpId = epId; + param->transport = pJob->transport; + + pMsgSendInfo->param = param; + pMsgSendInfo->fp = fp; + + SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo, .freeFunc = schFreeRpcCtxVal}; + if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { + SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, true)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosHashCleanup(pCtx->args); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); + + SCH_RET(code); +} + + +int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { + int32_t code = 0; + SSchHbTrans hb = {0}; + + hb.trans.transInst = pJob->transport; + + SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &hb.rpcCtx)); + + code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), &hb, sizeof(SSchHbTrans)); + if (code) { + schFreeRpcCtx(&hb.rpcCtx); + + if (HASH_NODE_EXIST(code)) { + *exist = true; + return TSDB_CODE_SUCCESS; + } + + qError("taosHashPut hb trans failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_ERR_RET(code); + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHeader **pDst) { + if (pSrc->isHbParam) { + SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam)); + if (NULL == dst) { + qError("malloc SSchHbCallbackParam failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(dst, pSrc, sizeof(*dst)); + *pDst = (SSchCallbackParamHeader *)dst; + + return TSDB_CODE_SUCCESS; + } + + SSchTaskCallbackParam *dst = taosMemoryMalloc(sizeof(SSchTaskCallbackParam)); + if (NULL == dst) { + qError("malloc SSchTaskCallbackParam failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(dst, pSrc, sizeof(*dst)); + *pDst = (SSchCallbackParamHeader *)dst; + + return TSDB_CODE_SUCCESS; +} + +int32_t schCloneSMsgSendInfo(void *src, void **dst) { + SMsgSendInfo *pSrc = src; + int32_t code = 0; + SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc)); + if (NULL == pDst) { + qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(pDst, pSrc, sizeof(*pSrc)); + pDst->param = NULL; + + SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); + + *dst = pDst; + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(pDst); + SCH_RET(code); +} + +int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { + int32_t code = 0; + memcpy(&pDst->brokenVal, &pSrc->brokenVal, sizeof(pSrc->brokenVal)); + pDst->brokenVal.val = NULL; + + SCH_ERR_RET(schCloneSMsgSendInfo(pSrc->brokenVal.val, &pDst->brokenVal.val)); + + pDst->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (NULL == pDst->args) { + qError("taosHashInit %d RpcCtx failed", 1); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SRpcCtxVal dst = {0}; + void *pIter = taosHashIterate(pSrc->args, NULL); + while (pIter) { + SRpcCtxVal *pVal = (SRpcCtxVal *)pIter; + int32_t *msgType = taosHashGetKey(pIter, NULL); + + dst = *pVal; + dst.val = NULL; + + SCH_ERR_JRET(schCloneSMsgSendInfo(pVal->val, &dst.val)); + + if (taosHashPut(pDst->args, msgType, sizeof(*msgType), &dst, sizeof(dst))) { + qError("taosHashPut msg %d to rpcCtx failed", *msgType); + (*dst.freeFunc)(dst.val); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pIter = taosHashIterate(pSrc->args, pIter); + } + + return TSDB_CODE_SUCCESS; + +_return: + + schFreeRpcCtx(pDst); + SCH_RET(code); +} + + +int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) { int32_t code = 0; SSchTrans *trans = (SSchTrans *)transport; - SMsgSendInfo *pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam)); + SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { - SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchCallbackParam)); + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1169,8 +1601,12 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - int64_t transporterId = 0; - code = asyncSendMsgToServer(trans->transInst, epSet, &transporterId, pMsgSendInfo); + qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", + TMSG_INFO(msgType), ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, + pJob->refId, trans->transInst, trans->transHandle); + + int64_t transporterId = 0; + code = asyncSendMsgToServerExt(trans->transInst, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx); if (code) { SCH_ERR_JRET(code); } @@ -1180,16 +1616,108 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet _return: - tfree(param); - tfree(pMsgSendInfo); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); + SCH_RET(code); +} + +int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { + SSchedulerHbReq req = {0}; + int32_t code = 0; + SRpcCtx rpcCtx = {0}; + SSchTrans trans = {0}; + int32_t msgType = TDMT_VND_QUERY_HEARTBEAT; + + req.header.vgId = htonl(nodeEpId->nodeId); + req.sId = schMgmt.sId; + memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); + + SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, nodeEpId->ep.port); + SCH_ERR_RET(code); + } + + SCH_LOCK(SCH_WRITE, &hb->lock); + code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); + memcpy(&trans, &hb->trans, sizeof(trans)); + SCH_UNLOCK(SCH_WRITE, &hb->lock); + + SCH_ERR_RET(code); + + int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); + if (msgSize < 0) { + qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + void *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + qError("calloc hb req %d failed", msgSize); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { + qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SMsgSendInfo *pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("calloc SMsgSendInfo failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); + if (NULL == param) { + qError("calloc SSchTaskCallbackParam failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); + + param->transport = trans.transInst; + + pMsgSendInfo->param = param; + pMsgSendInfo->msgInfo.pData = msg; + pMsgSendInfo->msgInfo.len = msgSize; + pMsgSendInfo->msgInfo.handle = trans.transHandle; + pMsgSendInfo->msgType = msgType; + pMsgSendInfo->fp = fp; + + int64_t transporterId = 0; + SEpSet epSet = {.inUse = 0, .numOfEps = 1}; + memcpy(&epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); + + qDebug("start to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d", trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port); + + code = asyncSendMsgToServerExt(trans.transInst, &epSet, &transporterId, pMsgSendInfo, true, &rpcCtx); + if (code) { + qError("fail to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d, error:%x - %s", + trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port, code, tstrerror(code)); + SCH_ERR_JRET(code); + } + + qDebug("hb msg sent"); + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(msg); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); + schFreeRpcCtx(&rpcCtx); SCH_RET(code); } int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void *msg = NULL; - int32_t code = 0; - bool isCandidateAddr = false; + void *msg = NULL; + int32_t code = 0; + bool isCandidateAddr = false; + bool persistHandle = false; + SRpcCtx rpcCtx = {0}; + if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; @@ -1201,7 +1729,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_CREATE_TABLE: case TDMT_VND_SUBMIT: { msgSize = pTask->msgLen; - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1212,10 +1740,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_QUERY: { + SCH_ERR_RET(schMakeQueryRpcCtx(pJob, pTask, &rpcCtx)); + uint32_t len = strlen(pJob->sql); - msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1233,12 +1762,14 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, memcpy(pMsg->msg, pJob->sql, len); memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); + + persistHandle = true; break; } case TDMT_VND_RES_READY: { msgSize = sizeof(SResReadyReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1255,7 +1786,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_FETCH: { msgSize = sizeof(SResFetchReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1268,11 +1799,12 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); pMsg->taskId = htobe64(pTask->taskId); + break; } case TDMT_VND_DROP_TASK: { msgSize = sizeof(STaskDropReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1289,6 +1821,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, break; } case TDMT_VND_QUERY_HEARTBEAT: { + SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &rpcCtx)); + SSchedulerHbReq req = {0}; req.sId = schMgmt.sId; req.header.vgId = addr->nodeId; @@ -1300,7 +1834,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_JOB_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1309,6 +1843,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + persistHandle = true; break; } default: @@ -1319,11 +1855,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_SET_TASK_LASTMSG_TYPE(pTask, msgType); - SSchTrans trans = {.transInst = pJob->transport, .transHandle = pTask ? pTask->handle : NULL}; - SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize)); + SSchTrans trans = {.transInst = pJob->transport, .transHandle = SCH_GET_TASK_HANDLE(pTask)}; + SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL))); - if (isCandidateAddr) { - SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); + if (msgType == TDMT_VND_QUERY) { + SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr, trans.transHandle)); } return TSDB_CODE_SUCCESS; @@ -1331,8 +1867,9 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, _return: SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + schFreeRpcCtx(&rpcCtx); - tfree(msg); + taosMemoryFreeClear(msg); SCH_RET(code); } @@ -1343,10 +1880,16 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { epId.nodeId = addr->nodeId; memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); +#if 1 SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); if (NULL == hb) { - SCH_ERR_RET(schBuildAndSendMsg(pJob, NULL, addr, TDMT_VND_QUERY_HEARTBEAT)); + bool exist = false; + SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId, &exist)); + if (!exist) { + SCH_ERR_RET(schBuildAndSendHbMsg(&epId)); + } } +#endif return TSDB_CODE_SUCCESS; } @@ -1358,11 +1901,17 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); if (schJobNeedToStop(pJob, &status)) { - SCH_TASK_DLOG("no need to launch task cause of job status, job status:%d", status); - + SCH_TASK_DLOG("no need to launch task cause of job status, job status:%s", jobTaskStatusStr(status)); + SCH_RET(atomic_load_32(&pJob->errCode)); } + // NOTE: race condition: the task should be put into the hash table before send msg to server + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { + SCH_ERR_RET(schPushTaskToExecList(pJob, pTask)); + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXECUTING); + } + SSubplan *plan = pTask->plan; if (NULL == pTask->msg) { // TODO add more detailed reason for failure @@ -1372,18 +1921,12 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { pTask->msgLen); SCH_ERR_RET(code); } else { - SCH_TASK_DLOG("physical plan len:%d, %s", pTask->msgLen, pTask->msg); + SCH_TASK_DLOGL("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } } SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); - // NOTE: race condition: the task should be put into the hash table before send msg to server - if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { - SCH_ERR_RET(schPushTaskToExecList(pJob, pTask)); - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXECUTING); - } - if (SCH_IS_QUERY_JOB(pJob)) { SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); } @@ -1398,6 +1941,8 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { bool enough = false; int32_t code = 0; + SCH_SET_TASK_HANDLE(pTask, NULL); + if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { SCH_ERR_JRET(schCheckIncTaskFlowQuota(pJob, pTask, &enough)); @@ -1438,36 +1983,39 @@ int32_t schLaunchJob(SSchJob *pJob) { } void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { - if (NULL == pTask->execAddrs) { - SCH_TASK_DLOG("no exec address, status:%d", SCH_GET_TASK_STATUS(pTask)); + if (NULL == pTask->execNodes) { + SCH_TASK_DLOG("no exec address, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; } - int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); + int32_t size = (int32_t)taosArrayGetSize(pTask->execNodes); if (size <= 0) { - SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; } - SQueryNodeAddr *addr = NULL; + SSchNodeInfo *nodeInfo = NULL; for (int32_t i = 0; i < size; ++i) { - addr = (SQueryNodeAddr *)taosArrayGet(pTask->execAddrs, i); + nodeInfo = (SSchNodeInfo *)taosArrayGet(pTask->execNodes, i); + SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); - schBuildAndSendMsg(pJob, pTask, addr, TDMT_VND_DROP_TASK); + schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_VND_DROP_TASK); } SCH_TASK_DLOG("task has %d exec address", size); } void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { + if (!SCH_IS_NEED_DROP_JOB(pJob)) { + return; + } + void *pIter = taosHashIterate(list, NULL); while (pIter) { SSchTask *pTask = *(SSchTask **)pIter; - if (!SCH_TASK_NO_NEED_DROP(pTask)) { - schDropTaskOnExecutedNode(pJob, pTask); - } + schDropTaskOnExecutedNode(pJob, pTask); pIter = taosHashIterate(list, pIter); } @@ -1523,10 +2071,9 @@ void schFreeJobImpl(void *job) { taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); - - tfree(pJob->res); - - tfree(pJob); + + taosMemoryFreeClear(pJob->resData); + taosMemoryFreeClear(pJob); qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); } @@ -1540,7 +2087,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD } int32_t code = 0; - SSchJob *pJob = calloc(1, sizeof(SSchJob)); + SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1595,11 +2142,11 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD *job = pJob->refId; if (syncSchedule) { - SCH_JOB_DLOG("will wait for rsp now, job status:%d", SCH_GET_JOB_STATUS(pJob)); + SCH_JOB_DLOG("will wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); tsem_wait(&pJob->rspSem); } - SCH_JOB_DLOG("job exec done, job status:%d", SCH_GET_JOB_STATUS(pJob)); + SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); schReleaseJob(pJob->refId); @@ -1662,8 +2209,10 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); SSchJob *job = schAcquireJob(*pJob); + pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; + schReleaseJob(*pJob); return TSDB_CODE_SUCCESS; @@ -1725,7 +2274,7 @@ int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSubQueryMsg* pMsg = calloc(1, msgSize); + SSubQueryMsg* pMsg = taosMemoryCalloc(1, msgSize); pMsg->header.vgId = tInfo.addr.nodeId; @@ -1742,7 +2291,7 @@ int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks) { if (NULL == taosArrayPush(info, &tInfo)) { qError("taosArrayPush failed, idx:%d", i); - free(msg); + taosMemoryFree(msg); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } @@ -1774,7 +2323,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { info.addr = src->addr; for (int32_t i = 0; i < copyNum; ++i) { - info.msg = malloc(msgSize); + info.msg = taosMemoryMalloc(msgSize); if (NULL == info.msg) { qError("malloc %d failed", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1786,7 +2335,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { if (NULL == taosArrayPush(*dst, &info)) { qError("taosArrayPush failed, idx:%d", i); - free(info.msg); + taosMemoryFree(info.msg); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } @@ -1816,13 +2365,13 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { int8_t status = SCH_GET_JOB_STATUS(pJob); if (status == JOB_TASK_STATUS_DROPPING) { - SCH_JOB_ELOG("job is dropping, status:%d", status); + SCH_JOB_ELOG("job is dropping, status:%s", jobTaskStatusStr(status)); schReleaseJob(job); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } if (!SCH_JOB_NEED_FETCH(pJob)) { - SCH_JOB_ELOG("no need to fetch data, status:%d", SCH_GET_JOB_STATUS(pJob)); + SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); schReleaseJob(job); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } @@ -1834,10 +2383,10 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { } if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { - SCH_JOB_ELOG("job failed or dropping, status:%d", status); + SCH_JOB_ELOG("job failed or dropping, status:%s", jobTaskStatusStr(status)); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } else if (status == JOB_TASK_STATUS_SUCCEED) { - SCH_JOB_DLOG("job already succeed, status:%d", status); + SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status)); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_ERR_JRET(schFetchFromRemote(pJob)); @@ -1848,17 +2397,17 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { status = SCH_GET_JOB_STATUS(pJob); if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { - SCH_JOB_ELOG("job failed or dropping, status:%d", status); + SCH_JOB_ELOG("job failed or dropping, status:%s", jobTaskStatusStr(status)); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } - - if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { + + if (pJob->resData && ((SRetrieveTableRsp *)pJob->resData)->completed) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } while (true) { - *pData = atomic_load_ptr(&pJob->res); - if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) { + *pData = atomic_load_ptr(&pJob->resData); + if (*pData != atomic_val_compare_exchange_ptr(&pJob->resData, *pData, NULL)) { continue; } @@ -1866,7 +2415,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { } if (NULL == *pData) { - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); if (rsp) { rsp->completed = 1; } @@ -1928,7 +2477,7 @@ void schedulerFreeTaskList(SArray *taskList) { int32_t taskNum = taosArrayGetSize(taskList); for (int32_t i = 0; i < taskNum; ++i) { STaskInfo *info = taosArrayGet(taskList, i); - tfree(info->msg); + taosMemoryFreeClear(info->msg); } taosArrayDestroy(taskList); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 376ed1e2bc1f5015bd5c8c2466ac77877cf53ce1..cf04b06579cf55aad894e8322e50d4d06d52aa5c 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -95,8 +95,8 @@ void schtBuildQueryDag(SQueryPlan *dag) { SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *scanPlan = (SSubplan *)calloc(1, sizeof(SSubplan)); - SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; @@ -110,7 +110,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); - scanPlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + scanPlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan->msgType = TDMT_VND_QUERY; mergePlan->id.queryId = qId; @@ -122,7 +122,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->pChildren = nodesMakeList(); mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_VND_QUERY; merge->pNodeList = nodesMakeList(); @@ -148,8 +148,8 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *scanPlan = (SSubplan *)calloc(scanPlanNum, sizeof(SSubplan)); - SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(scanPlanNum, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); merge->pNodeList = nodesMakeList(); scan->pNodeList = nodesMakeList(); @@ -173,7 +173,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan[i].pChildren = NULL; scanPlan[i].level = 1; scanPlan[i].pParents = nodesMakeList(); - scanPlan[i].pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + scanPlan[i].pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan[i].msgType = TDMT_VND_QUERY; nodesListAppend(scanPlan[i].pParents, (SNode*)mergePlan); @@ -190,7 +190,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_VND_QUERY; nodesListAppend(merge->pNodeList, (SNode*)mergePlan); @@ -213,7 +213,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { dag->pSubplans = nodesMakeList(); SNodeListNode *inserta = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *insertPlan = (SSubplan *)calloc(2, sizeof(SSubplan)); + SSubplan *insertPlan = (SSubplan *)taosMemoryCalloc(2, sizeof(SSubplan)); insertPlan[0].id.queryId = qId; insertPlan[0].id.groupId = 0x0000000000000003; @@ -228,7 +228,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; insertPlan[0].pNode = NULL; - insertPlan[0].pDataSink = (SDataSinkNode*)calloc(1, sizeof(SDataSinkNode)); + insertPlan[0].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[0].msgType = TDMT_VND_SUBMIT; insertPlan[1].id.queryId = qId; @@ -244,7 +244,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; insertPlan[1].pNode = NULL; - insertPlan[1].pDataSink = (SDataSinkNode*)calloc(1, sizeof(SDataSinkNode)); + insertPlan[1].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[1].msgType = TDMT_VND_SUBMIT; inserta->pNodeList = nodesMakeList(); @@ -258,7 +258,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { - *str = (char *)calloc(1, 20); + *str = (char *)taosMemoryCalloc(1, 20); *len = 20; return 0; } @@ -312,9 +312,9 @@ void schtSetRpcSendRequest() { int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) { if (pInfo) { - tfree(pInfo->param); - tfree(pInfo->msgInfo.pData); - free(pInfo); + taosMemoryFreeClear(pInfo->param); + taosMemoryFreeClear(pInfo->msgInfo.pData); + taosMemoryFree(pInfo); } return 0; } @@ -373,7 +373,7 @@ void *schtCreateFetchRspThread(void *param) { taosSsleep(1); int32_t code = 0; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; @@ -387,7 +387,7 @@ void *schtCreateFetchRspThread(void *param) { void *schtFetchRspThread(void *aa) { SDataBuf dataBuf = {0}; - SSchCallbackParam* param = NULL; + SSchTaskCallbackParam* param = NULL; while (!schtTestStop) { if (0 == atomic_val_compare_exchange_32(&schtStartFetch, 1, 0)) { @@ -396,13 +396,13 @@ void *schtFetchRspThread(void *aa) { taosUsleep(1); - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->queryId = schtQueryId; param->taskId = schtFetchTaskId; int32_t code = 0; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; @@ -449,7 +449,7 @@ void* schtRunJobThread(void *aa) { schtSetAsyncSendMsgToServer(); SSchJob *pJob = NULL; - SSchCallbackParam *param = NULL; + SSchTaskCallbackParam *param = NULL; SHashObj *execTasks = NULL; SDataBuf dataBuf = {0}; uint32_t jobFinished = 0; @@ -484,7 +484,7 @@ void* schtRunJobThread(void *aa) { pIter = taosHashIterate(pJob->execTasks, pIter); } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -504,7 +504,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -524,7 +524,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -544,7 +544,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -684,11 +684,11 @@ TEST(queryTest, normalCase) { pIter = taosHashIterate(pJob->execTasks, pIter); } - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, &job); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); void *data = NULL; code = schedulerFetchRows(job, &data); @@ -697,7 +697,7 @@ TEST(queryTest, normalCase) { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; ASSERT_EQ(pRsp->completed, 1); ASSERT_EQ(pRsp->numOfRows, 10); - tfree(data); + taosMemoryFreeClear(data); data = NULL; code = schedulerFetchRows(job, &data); @@ -713,6 +713,116 @@ TEST(queryTest, normalCase) { schedulerDestroy(); } +TEST(queryTest, readyFirstCase) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + SVgroupInfo vgInfo = {0}; + int64_t job = 0; + SQueryPlan dag; + + memset(&dag, 0, sizeof(dag)); + + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); + + SEp qnodeAddr = {0}; + strcpy(qnodeAddr.fqdn, "qnode0.ep"); + qnodeAddr.port = 6031; + taosArrayPush(qnodeList, &qnodeAddr); + + int32_t code = schedulerInit(NULL); + ASSERT_EQ(code, 0); + + schtBuildQueryDag(&dag); + + schtSetPlanToString(); + schtSetExecNode(); + schtSetAsyncSendMsgToServer(); + + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); + ASSERT_EQ(code, 0); + + + SSchJob *pJob = schAcquireJob(job); + + void *pIter = taosHashIterate(pJob->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SResReadyRsp rsp = {0}; + code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + printf("code:%d", code); + ASSERT_EQ(code, 0); + pIter = taosHashIterate(pJob->execTasks, pIter); + } + + pIter = taosHashIterate(pJob->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SQueryTableRsp rsp = {0}; + code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); + + ASSERT_EQ(code, 0); + pIter = taosHashIterate(pJob->execTasks, pIter); + } + + pIter = taosHashIterate(pJob->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SResReadyRsp rsp = {0}; + code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + ASSERT_EQ(code, 0); + + pIter = taosHashIterate(pJob->execTasks, pIter); + } + + pIter = taosHashIterate(pJob->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SQueryTableRsp rsp = {0}; + code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); + + ASSERT_EQ(code, 0); + pIter = taosHashIterate(pJob->execTasks, pIter); + } + + + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + + void *data = NULL; + code = schedulerFetchRows(job, &data); + ASSERT_EQ(code, 0); + + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; + ASSERT_EQ(pRsp->completed, 1); + ASSERT_EQ(pRsp->numOfRows, 10); + taosMemoryFreeClear(data); + + data = NULL; + code = schedulerFetchRows(job, &data); + ASSERT_EQ(code, 0); + ASSERT_TRUE(data == NULL); + + schReleaseJob(job); + + schedulerFreeJob(job); + + schtFreeQueryDag(&dag); + + schedulerDestroy(); +} + + + TEST(queryTest, flowCtrlCase) { void *mockPointer = (void *)0x1; char *clusterId = "cluster1"; @@ -780,11 +890,11 @@ TEST(queryTest, flowCtrlCase) { } - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, &job); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); void *data = NULL; code = schedulerFetchRows(job, &data); @@ -793,7 +903,7 @@ TEST(queryTest, flowCtrlCase) { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; ASSERT_EQ(pRsp->completed, 1); ASSERT_EQ(pRsp->numOfRows, 10); - tfree(data); + taosMemoryFreeClear(data); data = NULL; code = schedulerFetchRows(job, &data); @@ -834,11 +944,11 @@ TEST(insertTest, normalCase) { schtSetPlanToString(); schtSetAsyncSendMsgToServer(); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtSendRsp, &insertJobRefId); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", &res); @@ -851,13 +961,13 @@ TEST(insertTest, normalCase) { } TEST(multiThread, forceFree) { - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2, thread3; - pthread_create(&(thread1), &thattr, schtRunJobThread, NULL); - pthread_create(&(thread2), &thattr, schtFreeJobThread, NULL); - pthread_create(&(thread3), &thattr, schtFetchRspThread, NULL); + TdThread thread1, thread2, thread3; + taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL); + taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL); + taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL); while (true) { if (schtTestDeadLoop) { diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..572c70d31ba6140b4c368d3570e31436259ed862 --- /dev/null +++ b/source/libs/stream/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src STREAM_SRC) +add_library(stream STATIC ${STREAM_SRC}) +target_include_directories( + stream + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/stream" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + stream + PRIVATE os util transport qcom executor +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/libs/stream/inc/tstreamInc.h b/source/libs/stream/inc/tstreamInc.h new file mode 100644 index 0000000000000000000000000000000000000000..c96901e56754da81894c0c332e5cc3666d235e3b --- /dev/null +++ b/source/libs/stream/inc/tstreamInc.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _TSTREAM_H_ +#define _TSTREAM_H_ + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef _TSTREAM_H_ */ diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c new file mode 100644 index 0000000000000000000000000000000000000000..3ec660367363959b72ebd3c3332e30c0899c0fb8 --- /dev/null +++ b/source/libs/stream/src/tstream.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tstream.h" +#include "executor.h" + +int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) { + SArray* pRes = NULL; + // source + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK && pTask->sourceType != TASK_SOURCE__SCAN) return 0; + + // exec + if (pTask->execType != TASK_EXEC__NONE) { + ASSERT(workId < pTask->exec.numOfRunners); + void* exec = pTask->exec.runners[workId].executor; + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) { + return -1; + } + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + qSetStreamInput(exec, input, inputType); + while (1) { + SSDataBlock* output; + uint64_t ts; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) { + break; + } + taosArrayPush(pRes, output); + } + } else if (inputType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + const SArray* blocks = (const SArray*)input; + int32_t sz = taosArrayGetSize(blocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pBlock = taosArrayGet(blocks, i); + qSetStreamInput(exec, pBlock, inputType); + while (1) { + SSDataBlock* output; + uint64_t ts; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) { + break; + } + taosArrayPush(pRes, output); + } + } + } else { + ASSERT(0); + } + } else { + ASSERT(inputType == STREAM_DATA_TYPE_SSDATA_BLOCK); + pRes = (SArray*)input; + } + + // sink + if (pTask->sinkType == TASK_SINK__TABLE) { + // + } else if (pTask->sinkType == TASK_SINK__SMA) { + // + } else if (pTask->sinkType == TASK_SINK__FETCH) { + // + } else if (pTask->sinkType == TASK_SINK__SHOW) { + blockDebugShowData(pRes); + } else { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + } + + // dispatch + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + SStreamTaskExecReq req = { + .streamId = pTask->streamId, + .taskId = pTask->taskId, + .data = pRes, + }; + + int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req); + void* buf = rpcMallocCont(tlen); + + if (buf == NULL) { + return -1; + } + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSStreamTaskExecReq(&abuf, &req); + + SRpcMsg dispatchMsg = { + .pCont = buf, + .contLen = tlen, + .code = 0, + .msgType = pTask->dispatchMsgType, + }; + + int32_t qType; + if (pTask->dispatchMsgType == TDMT_VND_TASK_PIPE_EXEC || pTask->dispatchMsgType == TDMT_SND_TASK_PIPE_EXEC) { + qType = FETCH_QUEUE; + } else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC || + pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) { + qType = MERGE_QUEUE; + } else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) { + qType = WRITE_QUEUE; + } else { + ASSERT(0); + } + tmsgPutToQueue(pMsgCb, qType, &dispatchMsg); + + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + SStreamTaskExecReq req = { + .streamId = pTask->streamId, + .taskId = pTask->taskId, + .data = pRes, + }; + + int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req); + void* buf = rpcMallocCont(tlen); + + if (buf == NULL) { + return -1; + } + + ((SMsgHead*)buf)->vgId = htonl(pTask->fixedEpDispatcher.nodeId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSStreamTaskExecReq(&abuf, &req); + + SRpcMsg dispatchMsg = { + .pCont = buf, + .contLen = tlen, + .code = 0, + .msgType = pTask->dispatchMsgType, + }; + + SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; + + tmsgSendReq(pMsgCb, pEpSet, &dispatchMsg); + + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + // TODO + + } else { + ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); + } + return 0; +} + +int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->streamId); + tlen += taosEncodeFixedI32(buf, pReq->taskId); + tlen += tEncodeDataBlocks(buf, pReq->data); + return tlen; +} + +void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->streamId); + buf = taosDecodeFixedI32(buf, &pReq->taskId); + buf = tDecodeDataBlocks(buf, &pReq->data); + return (void*)buf; +} + +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { taosArrayDestroy(pReq->data); } + +SStreamTask* tNewSStreamTask(int64_t streamId) { + SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); + if (pTask == NULL) { + return NULL; + } + pTask->taskId = tGenIdPI32(); + pTask->streamId = streamId; + pTask->status = STREAM_TASK_STATUS__RUNNING; + /*pTask->qmsg = NULL;*/ + return pTask; +} + +int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask) { + /*if (tStartEncode(pEncoder) < 0) return -1;*/ + if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1; + if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->downstreamTaskId) < 0) return -1; + + if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + + if (pTask->execType != TASK_EXEC__NONE) { + if (tEncodeI8(pEncoder, pTask->exec.parallelizable) < 0) return -1; + if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; + } + + if (pTask->sinkType != TASK_SINK__NONE) { + // TODO: wrap + if (tEncodeI8(pEncoder, pTask->tbSink.reserved) < 0) return -1; + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + if (tEncodeI8(pEncoder, pTask->inplaceDispatcher.reserved) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1; + } + + /*tEndEncode(pEncoder);*/ + return pEncoder->pos; +} + +int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask) { + /*if (tStartDecode(pDecoder) < 0) return -1;*/ + if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1; + if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->downstreamTaskId) < 0) return -1; + + if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + + if (pTask->execType != TASK_EXEC__NONE) { + if (tDecodeI8(pDecoder, &pTask->exec.parallelizable) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; + } + + if (pTask->sinkType != TASK_SINK__NONE) { + if (tDecodeI8(pDecoder, &pTask->tbSink.reserved) < 0) return -1; + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + if (tDecodeI8(pDecoder, &pTask->inplaceDispatcher.reserved) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1; + } + + /*tEndDecode(pDecoder);*/ + return 0; +} + +void tFreeSStreamTask(SStreamTask* pTask) { + // TODO + /*taosMemoryFree(pTask->qmsg);*/ + /*taosMemoryFree(pTask->executor);*/ + /*taosMemoryFree(pTask);*/ +} + +#if 0 +int32_t tEncodeSStreamTaskExecReq(SCoder* pEncoder, const SStreamTaskExecReq* pReq) { + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + /*if (tEncodeDataBlocks(buf, pReq->streamId) < 0) return -1;*/ + return pEncoder->size; +} +int32_t tDecodeSStreamTaskExecReq(SCoder* pDecoder, SStreamTaskExecReq* pReq) { + return pEncoder->size; +} +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { + taosArrayDestroyEx(pReq->data, tDeleteSSDataBlock); +} +#endif diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 66c7c8620df10728ef54c7ccf08c5e854d493f3b..7940d7f436335c10ec0745b98e96ab3142186810 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -31,10 +31,10 @@ extern "C" { #define TIMER_MAX_MS 0x7FFFFFFF #define ENV_TICK_TIMER_MS 1000 #define PING_TIMER_MS 1000 -#define ELECT_TIMER_MS_MIN 1500 -#define ELECT_TIMER_MS_MAX 3000 +#define ELECT_TIMER_MS_MIN 150 +#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 300 +#define HEARTBEAT_TIMER_MS 30 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 5a4b7555bf630bad4fc537eae48c3ddce0b8689e..99f9deb99e8da9e484d07e3566995dd1f3e00daf 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -35,7 +35,7 @@ extern "C" { typedef struct SSyncIO { STaosQueue *pMsgQ; STaosQset * pQset; - pthread_t consumerTid; + TdThread consumerTid; void * serverRpc; void * clientRpc; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8e36424f192e03fc72b6878f184ce075ecbdd0bc..cfa87048a713d735bb0c8f09ee08a7fcedf590f3 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -142,7 +142,6 @@ typedef struct SSyncNode { SRaftId leaderCache; // life cycle - int32_t refCount; int64_t rid; // tla+ server vars @@ -161,6 +160,11 @@ typedef struct SSyncNode { SSyncLogStore* pLogStore; SyncIndex commitIndex; + // timer ms init + int32_t pingBaseLine; + int32_t electBaseLine; + int32_t hbBaseLine; + // ping timer tmr_h pPingTimer; int32_t pingTimerMS; @@ -218,10 +222,12 @@ int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- -int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); -cJSON* syncNode2Json(const SSyncNode* pSyncNode); -char* syncNode2Str(const SSyncNode* pSyncNode); +int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); +int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); +cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); +SSyncNode* syncNodeAcquire(int64_t rid); +void syncNodeRelease(SSyncNode* pNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 037a49f45c5e7a43f77d2dfff13edf7be56398fd..d4e63e9676d519c0ae92a0d93a7199f780c58eee 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -27,6 +27,12 @@ extern "C" { #include "syncMessage.h" #include "taosdef.h" +typedef enum EntryType { + SYNC_RAFT_ENTRY_NOOP = 0, + SYNC_RAFT_ENTRY_DATA = 1, + SYNC_RAFT_ENTRY_CONFIG = 2, +} EntryType; + typedef struct SSyncRaftEntry { uint32_t bytes; uint32_t msgType; @@ -35,12 +41,15 @@ typedef struct SSyncRaftEntry { bool isWeak; SyncTerm term; SyncIndex index; + EntryType entryType; uint32_t dataLen; char data[]; } SSyncRaftEntry; SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 +SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType); +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index); void syncEntryDestory(SSyncRaftEntry* pEntry); char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index d979e0df150f48ec403012d13eafb5431c5b42e6..2196d6c207430b166d6596344b9c5837f2b03d28 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -47,6 +47,8 @@ SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); cJSON* logStore2Json(SSyncLogStore* pLogStore); char* logStore2Str(SSyncLogStore* pLogStore); +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); +char* logStoreSimple2Str(SSyncLogStore* pLogStore); // for debug void logStorePrint(SSyncLogStore* pLogStore); @@ -54,6 +56,11 @@ void logStorePrint2(char* s, SSyncLogStore* pLogStore); void logStoreLog(SSyncLogStore* pLogStore); void logStoreLog2(char* s, SSyncLogStore* pLogStore); +void logStoreSimplePrint(SSyncLogStore* pLogStore); +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreSimpleLog(SSyncLogStore* pLogStore); +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 1b702c252867dc8a909048d2bea2c960989f40b4..a4bb11afc63e723b29b2de438081a928f4a8366c 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -44,7 +44,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest); // ---- misc ---- int32_t syncUtilRand(int32_t max); -int32_t syncUtilElectRandomMS(); +int32_t syncUtilElectRandomMS(int32_t min, int32_t max); int32_t syncUtilQuorum(int32_t replicaNum); cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 270180e3477b4c6359feb07a918de56ac097fcef..2e9cd63de6aa0f80daefa3f385c9b4463e4f5adc 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -87,7 +87,10 @@ // int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; - syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesCb== term:%lu", ths->pRaftStore->currentTerm); + syncAppendEntriesLog2(logBuf, pMsg); if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); @@ -117,6 +120,11 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // reject request if ((pMsg->term < ths->pRaftStore->currentTerm) || ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + sTrace( + "syncNodeOnAppendEntriesCb --> reject, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " + "logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; @@ -134,6 +142,11 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // return to follower state if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + sTrace( + "syncNodeOnAppendEntriesCb --> return to follower, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "ths->state:%d, logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + syncNodeBecomeFollower(ths); // ret or reply? @@ -142,89 +155,63 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // accept request if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { - bool preMatch = false; - if (pMsg->prevLogIndex == SYNC_INDEX_INVALID && - ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) { - preMatch = true; - } - if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { - SSyncRaftEntry* pPreEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogIndex); - assert(pPreEntry != NULL); - if (pMsg->prevLogTerm == pPreEntry->term) { - preMatch = true; - } - syncEntryDestory(pPreEntry); - } + // preIndex = -1, or has preIndex entry in local log + assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); - if (preMatch) { - // must has preIndex in local log - assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + // has extra entries (> preIndex) in local log + bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); - bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); - bool hasAppendEntries = pMsg->dataLen > 0; + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; - if (hasExtraEntries && hasAppendEntries) { - // conflict - bool conflict = false; + sTrace( + "syncNodeOnAppendEntriesCb --> accept, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " + "logOK:%d, hasExtraEntries:%d, hasAppendEntries:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries); - SyncIndex extraIndex = pMsg->prevLogIndex + 1; - SSyncRaftEntry* pExtraEntry = logStoreGetEntry(ths->pLogStore, extraIndex); - assert(pExtraEntry != NULL); + if (hasExtraEntries && hasAppendEntries) { + // not conflict by default + bool conflict = false; - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = logStoreGetEntry(ths->pLogStore, extraIndex); + assert(pExtraEntry != NULL); - assert(extraIndex == pAppendEntry->index); - if (pExtraEntry->term == pAppendEntry->term) { - conflict = true; - } + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); - if (conflict) { - // roll back - SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); - SyncIndex delEnd = extraIndex; + // log not match, conflict + assert(extraIndex == pAppendEntry->index); + if (pExtraEntry->term != pAppendEntry->term) { + conflict = true; + } - // notice! reverse roll back! - for (SyncIndex index = delEnd; index >= delBegin; --index) { - if (ths->pFsm->FpRollBackCb != NULL) { - SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); - assert(pRollBackEntry != NULL); + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); - ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0); - rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pRollBackEntry); - } - } + sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); - // delete confict entries - ths->pLogStore->truncate(ths->pLogStore, extraIndex); + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); + assert(pRollBackEntry != NULL); - // append new entries - ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + // maybe is a NOOP ENTRY + // assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA); - // pre commit - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); - if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0); - } + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0, ths->state); + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pRollBackEntry); } - rpcFreeCont(rpcMsg.pCont); } - // free memory - syncEntryDestory(pExtraEntry); - syncEntryDestory(pAppendEntry); - - } else if (hasExtraEntries && !hasAppendEntries) { - // do nothing - - } else if (!hasExtraEntries && hasAppendEntries) { - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); // append new entries ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); @@ -233,54 +220,64 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0); + if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 2, ths->state); } } rpcFreeCont(rpcMsg.pCont); + } - // free memory - syncEntryDestory(pAppendEntry); + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); - } else if (!hasExtraEntries && !hasAppendEntries) { - // do nothing + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing - } else { - assert(0); - } + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = true; + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); - if (hasAppendEntries) { - pReply->matchIndex = pMsg->prevLogIndex + 1; - } else { - pReply->matchIndex = pMsg->prevLogIndex; + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 3, ths->state); + } } + rpcFreeCont(rpcMsg.pCont); - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + // free memory + syncEntryDestory(pAppendEntry); - syncAppendEntriesReplyDestroy(pReply); + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing } else { - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = false; - pReply->matchIndex = SYNC_INDEX_INVALID; + assert(0); + } - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); - syncAppendEntriesReplyDestroy(pReply); + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; } + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + // maybe update commit index from leader if (pMsg->commitIndex > ths->commitIndex) { // has commit entry in local @@ -304,8 +301,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (ths->pFsm->FpCommitCb != NULL) { - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + if (ths->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 9db9a3e8ac532790dbcffff40f80b5950adc983c..790ac7f8e1c997c3f8721b5badfc85a9b6fa2a94 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -37,7 +37,10 @@ // int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; - syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplyCb== term:%lu", ths->pRaftStore->currentTerm); + syncAppendEntriesReplyLog2(logBuf, pMsg); if (pMsg->term < ths->pRaftStore->currentTerm) { sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, @@ -45,6 +48,9 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pMatchIndex", ths->pMatchIndex); + // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { // syncNodeUpdateTerm(ths, pMsg->term); @@ -74,5 +80,8 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pMatchIndex", ths->pMatchIndex); + return ret; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 38b39e04cf06b4dfbb6cc1f1da15756f7b50affe..6023c00afa183716c06828470a1e91e0fdec087b 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -50,8 +50,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // update commit index SyncIndex newCommitIndex = pSyncNode->commitIndex; for (SyncIndex index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); index > pSyncNode->commitIndex; - ++index) { + --index) { bool agree = syncAgree(pSyncNode, index); + sTrace("syncMaybeAdvanceCommitIndex syncAgree:%d, index:%ld, pSyncNode->commitIndex:%ld", agree, index, + pSyncNode->commitIndex); if (agree) { // term SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); @@ -61,7 +63,14 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (pEntry->term == pSyncNode->pRaftStore->currentTerm) { // update commit index newCommitIndex = index; + sTrace("syncMaybeAdvanceCommitIndex maybe to update, newCommitIndex:%ld commit, pSyncNode->commitIndex:%ld", + newCommitIndex, pSyncNode->commitIndex); break; + } else { + sTrace( + "syncMaybeAdvanceCommitIndex can not commit due to term not equal, pEntry->term:%lu, " + "pSyncNode->pRaftStore->currentTerm:%lu", + pEntry->term, pSyncNode->pRaftStore->currentTerm); } } } @@ -70,6 +79,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SyncIndex beginIndex = pSyncNode->commitIndex + 1; SyncIndex endIndex = newCommitIndex; + sTrace("syncMaybeAdvanceCommitIndex sync commit %ld", newCommitIndex); + // update commit index pSyncNode->commitIndex = newCommitIndex; @@ -86,8 +97,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (pSyncNode->pFsm->FpCommitCb != NULL) { - pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, pSyncNode->state); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index d47a525e2584652a700b2a9d4da2721fa0571dc2..726ebdae07de5e406ac488cce1660524f1310112 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -74,7 +74,7 @@ static void syncEnvTick(void *param, void *tmrId) { } static SSyncEnv *doSyncEnvStart() { - SSyncEnv *pSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + SSyncEnv *pSyncEnv = (SSyncEnv *)taosMemoryMalloc(sizeof(SSyncEnv)); assert(pSyncEnv != NULL); memset(pSyncEnv, 0, sizeof(SSyncEnv)); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index c841c2ac068957bcb82613647cc3ec95a9c7ac2e..717b1cc7565645eec916ec5f677cf85e4b8bb2e3 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void * syncIOConsumerFunc(void *param); +static void *syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -75,6 +75,7 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { syncRpcMsgPrint2(logBuf, pMsg); pMsg->handle = NULL; + pMsg->noResp = 1; rpcSendRequest(clientRpc, pEpSet, pMsg, NULL); return ret; } @@ -120,7 +121,7 @@ int32_t syncIOPingTimerStop() { // local function ------------ static SSyncIO *syncIOCreate(char *host, uint16_t port) { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + SSyncIO *io = (SSyncIO *)taosMemoryMalloc(sizeof(SSyncIO)); memset(io, 0, sizeof(*io)); io->pMsgQ = taosOpenQueue(); @@ -211,7 +212,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { // start consumer thread { - if (pthread_create(&io->consumerTid, NULL, syncIOConsumerFunc, io) != 0) { + if (taosThreadCreate(&io->consumerTid, NULL, syncIOConsumerFunc, io) != 0) { sError("failed to create sync consumer thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -228,15 +229,15 @@ static int32_t syncIOStartInternal(SSyncIO *io) { static int32_t syncIOStopInternal(SSyncIO *io) { int32_t ret = 0; atomic_store_8(&io->isStart, 0); - pthread_join(io->consumerTid, NULL); + taosThreadJoin(io->consumerTid, NULL); taosTmrCleanUp(io->timerMgr); return ret; } static void *syncIOConsumerFunc(void *param) { - SSyncIO * io = param; + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -257,13 +258,6 @@ static void *syncIOConsumerFunc(void *param) { assert(pSyncMsg != NULL); io->FpOnSyncPing(io->pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); - /* - pSyncMsg = syncPingBuild(pRpcMsg->contLen); - syncPingFromRpcMsg(pRpcMsg, pSyncMsg); - // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); - io->FpOnSyncPing(io->pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); - */ } } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { @@ -331,19 +325,21 @@ static void *syncIOConsumerFunc(void *param) { taosGetQitem(qall, (void **)&pRpcMsg); rpcFreeCont(pRpcMsg->pCont); - if (pRpcMsg->handle != NULL) { - int msgSize = 32; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.msgType = SYNC_RESPONSE; - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); - rpcMsg.handle = pRpcMsg->handle; - rpcMsg.code = 0; - - syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); - rpcSendResponse(&rpcMsg); - } + /* + if (pRpcMsg->handle != NULL) { + int msgSize = 32; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.msgType = SYNC_RESPONSE; + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + + syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); + rpcSendResponse(&rpcMsg); + } + */ taosFreeQitem(pRpcMsg); } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index aa97104180c37dfb2455a79cfef8696717ac5ec1..d33075054a888a1b2903fe230f6503e6c19aa580 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -19,7 +19,7 @@ // SMatchIndex ----------------------------- SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { - SSyncIndexMgr *pSyncIndexMgr = malloc(sizeof(SSyncIndexMgr)); + SSyncIndexMgr *pSyncIndexMgr = taosMemoryMalloc(sizeof(SSyncIndexMgr)); assert(pSyncIndexMgr != NULL); memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr)); @@ -33,7 +33,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { if (pSyncIndexMgr != NULL) { - free(pSyncIndexMgr); + taosMemoryFree(pSyncIndexMgr); } } @@ -78,12 +78,12 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); } int respondNum = 0; - int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { arr[i] = pSyncIndexMgr->index[i]; } cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "index", pIndex); snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); @@ -106,24 +106,24 @@ void syncIndexMgrPrint(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); printf("syncIndexMgrPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrPrint2(char *s, SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); printf("syncIndexMgrPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrLog(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); sTrace("syncIndexMgrLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrLog2(char *s, SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 6ed922962540d6d2327978a39335ef9bcb5ac2bd..78e454309ac24edac140d52434cc3f2d192916d8 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -31,44 +31,73 @@ #include "syncTimeout.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "tref.h" static int32_t tsNodeRefId = -1; // ------ local funciton --------- // enqueue message ---- -static void syncNodeEqPingTimer(void* param, void* tmrId); -static void syncNodeEqElectTimer(void* param, void* tmrId); -static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); +static void syncNodeEqPingTimer(void* param, void* tmrId); +static void syncNodeEqElectTimer(void* param, void* tmrId); +static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); +static int32_t syncNodeEqNoop(SSyncNode* ths); +static int32_t syncNodeAppendNoop(SSyncNode* ths); -// on message ---- +// process message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); + +// life cycle +static void syncFreeNode(void* param); // --------------------------------- int32_t syncInit() { - int32_t ret = syncEnvStart(); + int32_t ret; + tsNodeRefId = taosOpenRef(200, syncFreeNode); + if (tsNodeRefId < 0) { + sError("failed to init node ref"); + syncCleanUp(); + ret = -1; + } else { + ret = syncEnvStart(); + } + return ret; } void syncCleanUp() { int32_t ret = syncEnvStop(); assert(ret == 0); + + if (tsNodeRefId != -1) { + taosCloseRef(tsNodeRefId); + tsNodeRefId = -1; + } } int64_t syncStart(const SSyncInfo* pSyncInfo) { - int32_t ret = 0; SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); assert(pSyncNode != NULL); - // todo : return ref id - return ret; + pSyncNode->rid = taosAddRef(tsNodeRefId, pSyncNode); + if (pSyncNode->rid < 0) { + syncFreeNode(pSyncNode); + return -1; + } + + return pSyncNode->rid; } void syncStop(int64_t rid) { - // todo : get pointer from rid - SSyncNode* pSyncNode = NULL; + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return; + } syncNodeClose(pSyncNode); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + taosRemoveRef(tsNodeRefId, rid); } int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { @@ -76,13 +105,35 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return ret; } +int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + int32_t ret = syncPropose2(rid, pMsg, isWeak, 0); + return ret; +} + int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { - int32_t ret = 0; + int32_t ret = syncPropose(rid, pMsg, isWeak); + return ret; +} + +ESyncState syncGetMyRole(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); + return pSyncNode->state; +} + +int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum) { + int32_t ret = 0; + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return -1; + } + assert(rid == pSyncNode->rid); - // todo : get pointer from rid - SSyncNode* pSyncNode = NULL; if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak); + SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak); SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); @@ -90,23 +141,17 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { ret = 0; } else { - sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state)); + sTrace("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); ret = -1; // todo : need define err code !! } - return ret; -} -ESyncState syncGetMyRole(int64_t rid) { - // todo : get pointer from rid - SSyncNode* pSyncNode = NULL; - return pSyncNode->state; + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; } -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} - // open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { - SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); @@ -155,7 +200,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum); pSyncNode->leaderCache = EMPTY_RAFT_ID; - // init life cycle + // init life cycle outside // TLA+ Spec // InitHistoryVars == /\ elections = {} @@ -203,9 +248,14 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { assert(pSyncNode->pLogStore != NULL); pSyncNode->commitIndex = SYNC_INDEX_INVALID; + // timer ms init + pSyncNode->pingBaseLine = PING_TIMER_MS; + pSyncNode->electBaseLine = ELECT_TIMER_MS_MIN; + pSyncNode->hbBaseLine = HEARTBEAT_TIMER_MS; + // init ping timer pSyncNode->pPingTimer = NULL; - pSyncNode->pingTimerMS = PING_TIMER_MS; + pSyncNode->pingTimerMS = pSyncNode->pingBaseLine; atomic_store_64(&pSyncNode->pingTimerLogicClock, 0); atomic_store_64(&pSyncNode->pingTimerLogicClockUser, 0); pSyncNode->FpPingTimerCB = syncNodeEqPingTimer; @@ -213,7 +263,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init elect timer pSyncNode->pElectTimer = NULL; - pSyncNode->electTimerMS = syncUtilElectRandomMS(); + pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); atomic_store_64(&pSyncNode->electTimerLogicClock, 0); atomic_store_64(&pSyncNode->electTimerLogicClockUser, 0); pSyncNode->FpElectTimerCB = syncNodeEqElectTimer; @@ -221,7 +271,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init heartbeat timer pSyncNode->pHeartbeatTimer = NULL; - pSyncNode->heartbeatTimerMS = HEARTBEAT_TIMER_MS; + pSyncNode->heartbeatTimerMS = pSyncNode->hbBaseLine; atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, 0); atomic_store_64(&pSyncNode->heartbeatTimerLogicClockUser, 0); pSyncNode->FpHeartbeatTimerCB = syncNodeEqHeartbeatTimer; @@ -260,7 +310,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); - free(pSyncNode); + taosMemoryFree(pSyncNode); } // ping -------------- @@ -355,7 +405,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { int32_t ret = 0; - int32_t electMS = syncUtilElectRandomMS(); + int32_t electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); ret = syncNodeRestartElectTimer(pSyncNode, electMS); return ret; } @@ -444,6 +494,10 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); + // life cycle + snprintf(u64buf, sizeof(u64buf), "%ld", pSyncNode->rid); + cJSON_AddStringToObject(pRoot, "rid", u64buf); + // tla+ server vars cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); @@ -532,6 +586,17 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } +SSyncNode* syncNodeAcquire(int64_t rid) { + SSyncNode* pNode = taosAcquireRef(tsNodeRefId, rid); + if (pNode == NULL) { + sTrace("failed to acquire node from refId:%" PRId64, rid); + } + + return pNode; +} + +void syncNodeRelease(SSyncNode* pNode) { taosReleaseRef(tsNodeRefId, pNode->rid); } + // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { @@ -606,6 +671,10 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); assert(voteGrantedMajority(pSyncNode->pVotesGranted)); syncNodeBecomeLeader(pSyncNode); + + // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); + // syncNodeEqNoop(pSyncNode); } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { @@ -654,26 +723,26 @@ void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); printf("syncNodePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncNodePrint2(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); printf("syncNodePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncNodeLog(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncNodeLog2(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ------ local funciton --------- @@ -709,7 +778,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { syncTimeoutDestroy(pSyncMsg); // reset timer ms - pSyncNode->electTimerMS = syncUtilElectRandomMS(); + pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { @@ -740,6 +809,47 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } } +static int32_t syncNodeEqNoop(SSyncNode* ths) { + int32_t ret = 0; + assert(ths->state == TAOS_SYNC_STATE_LEADER); + + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + assert(pEntry != NULL); + + uint32_t entryLen; + char* serialized = syncEntrySerialize(pEntry, &entryLen); + SyncClientRequest* pSyncMsg = syncClientRequestBuild(entryLen); + assert(pSyncMsg->dataLen == entryLen); + memcpy(pSyncMsg->data, serialized, entryLen); + + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); + ths->FpEqMsg(ths->queue, &rpcMsg); + + taosMemoryFree(serialized); + syncClientRequestDestroy(pSyncMsg); + + return ret; +} + +static int32_t syncNodeAppendNoop(SSyncNode* ths) { + int32_t ret = 0; + + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + assert(pEntry != NULL); + + if (ths->state == TAOS_SYNC_STATE_LEADER) { + ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + syncNodeReplicate(ths); + } + + return ret; +} + // on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { int32_t ret = 0; @@ -788,8 +898,8 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); } } rpcFreeCont(rpcMsg.pCont); @@ -803,8 +913,8 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -1); + if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 1, ths->state); } } rpcFreeCont(rpcMsg.pCont); @@ -813,3 +923,10 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntryDestory(pEntry); return ret; } + +static void syncFreeNode(void* param) { + SSyncNode* pNode = param; + syncNodePrint2((char*)"==syncFreeNode==", pNode); + + taosMemoryFree(pNode); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 3ac4a7a1c083485fe776a92a59c2989b111966a7..5dca165455240d9eaf57c8cd7e76b264fbb68f3e 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -70,20 +70,20 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont2", s); - free(s); + taosMemoryFree(s); } else { pRoot = cJSON_CreateObject(); char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont2", s); - free(s); + taosMemoryFree(s); } cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); @@ -118,32 +118,32 @@ void syncRpcMsgPrint(SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); printf("syncRpcMsgPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); printf("syncRpcMsgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgLog(SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); sTrace("syncRpcMsgLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncTimeout---- SyncTimeout* syncTimeoutBuild() { uint32_t bytes = sizeof(SyncTimeout); - SyncTimeout* pMsg = malloc(bytes); + SyncTimeout* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_TIMEOUT; @@ -161,7 +161,7 @@ SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock void syncTimeoutDestroy(SyncTimeout* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -176,7 +176,7 @@ void syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg) { } char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncTimeoutSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -187,7 +187,7 @@ char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len) { SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncTimeout* pMsg = malloc(bytes); + SyncTimeout* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncTimeoutDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -243,32 +243,32 @@ void syncTimeoutPrint(const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); printf("syncTimeoutPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); printf("syncTimeoutPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutLog(const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); sTrace("syncTimeoutLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutLog2(char* s, const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncPing---- SyncPing* syncPingBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncPing) + dataLen; - SyncPing* pMsg = malloc(bytes); + SyncPing* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_PING; @@ -292,7 +292,7 @@ SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId) { void syncPingDestroy(SyncPing* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -308,7 +308,7 @@ void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { } char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncPingSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -319,7 +319,7 @@ char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len) { SyncPing* syncPingDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncPing* pMsg = malloc(bytes); + SyncPing* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncPingDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -385,10 +385,10 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -408,32 +408,32 @@ void syncPingPrint(const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); printf("syncPingPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingPrint2(char* s, const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); printf("syncPingPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingLog(const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); sTrace("syncPingLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncPingLog2(char* s, const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncPingReply---- SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncPingReply) + dataLen; - SyncPingReply* pMsg = malloc(bytes); + SyncPingReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_PING_REPLY; @@ -457,7 +457,7 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) void syncPingReplyDestroy(SyncPingReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -473,7 +473,7 @@ void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg } char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncPingReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -484,7 +484,7 @@ char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len) { SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncPingReply* pMsg = malloc(bytes); + SyncPingReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncPingReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -550,10 +550,10 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -573,32 +573,32 @@ void syncPingReplyPrint(const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); printf("syncPingReplyPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); printf("syncPingReplyPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyLog(const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); sTrace("syncPingReplyLog | len:%zu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyLog2(char* s, const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncClientRequest---- SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncClientRequest) + dataLen; - SyncClientRequest* pMsg = malloc(bytes); + SyncClientRequest* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_CLIENT_REQUEST; @@ -620,7 +620,7 @@ SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint6 void syncClientRequestDestroy(SyncClientRequest* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -635,7 +635,7 @@ void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientReque } char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncClientRequestSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -646,7 +646,7 @@ char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncClientRequest* pMsg = malloc(bytes); + SyncClientRequest* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncClientRequestDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -688,10 +688,10 @@ cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -711,32 +711,32 @@ void syncClientRequestPrint(const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); printf("syncClientRequestPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); printf("syncClientRequestPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestLog(const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); sTrace("syncClientRequestLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncRequestVote---- SyncRequestVote* syncRequestVoteBuild() { uint32_t bytes = sizeof(SyncRequestVote); - SyncRequestVote* pMsg = malloc(bytes); + SyncRequestVote* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_REQUEST_VOTE; @@ -745,7 +745,7 @@ SyncRequestVote* syncRequestVoteBuild() { void syncRequestVoteDestroy(SyncRequestVote* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -760,7 +760,7 @@ void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* } char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncRequestVoteSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -771,7 +771,7 @@ char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len) { SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncRequestVote* pMsg = malloc(bytes); + SyncRequestVote* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncRequestVoteDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -834,7 +834,7 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastLogIndex); cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf); @@ -857,32 +857,32 @@ void syncRequestVotePrint(const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); printf("syncRequestVotePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); printf("syncRequestVotePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteLog(const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); sTrace("syncRequestVoteLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncRequestVoteReply---- SyncRequestVoteReply* syncRequestVoteReplyBuild() { uint32_t bytes = sizeof(SyncRequestVoteReply); - SyncRequestVoteReply* pMsg = malloc(bytes); + SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_REQUEST_VOTE_REPLY; @@ -891,7 +891,7 @@ SyncRequestVoteReply* syncRequestVoteReplyBuild() { void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -906,7 +906,7 @@ void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestV } char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncRequestVoteReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -917,7 +917,7 @@ char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncRequestVoteReply* pMsg = malloc(bytes); + SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncRequestVoteReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1000,32 +1000,32 @@ void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); printf("syncRequestVoteReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); printf("syncRequestVoteReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); sTrace("syncRequestVoteReplyLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncAppendEntries---- SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncAppendEntries) + dataLen; - SyncAppendEntries* pMsg = malloc(bytes); + SyncAppendEntries* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_APPEND_ENTRIES; @@ -1035,7 +1035,7 @@ SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -1051,7 +1051,7 @@ void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntri } char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncAppendEntriesSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -1062,7 +1062,7 @@ char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len) SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncAppendEntries* pMsg = malloc(bytes); + SyncAppendEntries* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncAppendEntriesDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1127,23 +1127,23 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); - cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->commitIndex); - cJSON_AddStringToObject(pRoot, "commit_index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -1163,32 +1163,32 @@ void syncAppendEntriesPrint(const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); printf("syncAppendEntriesPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); printf("syncAppendEntriesPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesLog(const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); sTrace("syncAppendEntriesLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncAppendEntriesReply---- SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { uint32_t bytes = sizeof(SyncAppendEntriesReply); - SyncAppendEntriesReply* pMsg = malloc(bytes); + SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY; @@ -1197,7 +1197,7 @@ SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -1212,7 +1212,7 @@ void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppend } char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncAppendEntriesReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -1223,7 +1223,7 @@ char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint3 SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncAppendEntriesReply* pMsg = malloc(bytes); + SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncAppendEntriesReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1288,7 +1288,7 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "success", pMsg->success); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->matchIndex); cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); } @@ -1309,24 +1309,24 @@ void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); printf("syncAppendEntriesReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); printf("syncAppendEntriesReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); sTrace("syncAppendEntriesReplyLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index e4c5882e9896d423709e2fcc4c8de6245ad908b4..3de39a6aa802d192c2e7ae77c5eb9a57fc9b0a33 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -18,7 +18,7 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen; - SSyncRaftEntry* pEntry = malloc(bytes); + SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); assert(pEntry != NULL); memset(pEntry, 0, bytes); pEntry->bytes = bytes; @@ -28,6 +28,13 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { // step 4. SyncClientRequest => SSyncRaftEntry, add term, index SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index, SYNC_RAFT_ENTRY_DATA); + assert(pEntry != NULL); + + return pEntry; +} + +SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); assert(pEntry != NULL); @@ -37,21 +44,32 @@ SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncInde pEntry->isWeak = pMsg->isWeak; pEntry->term = term; pEntry->index = index; + pEntry->entryType = entryType; pEntry->dataLen = pMsg->dataLen; memcpy(pEntry->data, pMsg->data, pMsg->dataLen); return pEntry; } +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild(0); + assert(pEntry != NULL); + pEntry->term = term; + pEntry->index = index; + pEntry->entryType = SYNC_RAFT_ENTRY_NOOP; + + return pEntry; +} + void syncEntryDestory(SSyncRaftEntry* pEntry) { if (pEntry != NULL) { - free(pEntry); + taosMemoryFree(pEntry); } } // step 5. SSyncRaftEntry => bin, to raft log char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { - char* buf = malloc(pEntry->bytes); + char* buf = taosMemoryMalloc(pEntry->bytes); assert(buf != NULL); memcpy(buf, pEntry, pEntry->bytes); if (len != NULL) { @@ -63,7 +81,7 @@ char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { // step 6. bin => SSyncRaftEntry, from raft log SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SSyncRaftEntry* pEntry = malloc(bytes); + SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); assert(pEntry != NULL); memcpy(pEntry, buf, len); assert(len == pEntry->bytes); @@ -85,16 +103,17 @@ cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index); cJSON_AddStringToObject(pRoot, "index", u64buf); + cJSON_AddNumberToObject(pRoot, "entryType", pEntry->entryType); cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); char* s; s = syncUtilprintBin((char*)(pEntry->data), pEntry->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pEntry->data), pEntry->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -123,24 +142,24 @@ void syncEntryPrint(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncEntryLog(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 620e923629863a844d3b1382d43e341426e10cc7..63c6265d10672a321d4f9ae7e26e4887cde9384f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -17,10 +17,10 @@ #include "wal.h" SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { - SSyncLogStore* pLogStore = malloc(sizeof(SSyncLogStore)); + SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore)); assert(pLogStore != NULL); - pLogStore->data = malloc(sizeof(SSyncLogStoreData)); + pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData)); assert(pLogStore->data != NULL); SSyncLogStoreData* pData = pLogStore->data; @@ -34,13 +34,13 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->getLastTerm = logStoreLastTerm; pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex; - return pLogStore; // to avoid compiler error + return pLogStore; } void logStoreDestory(SSyncLogStore* pLogStore) { if (pLogStore != NULL) { - free(pLogStore->data); - free(pLogStore); + taosMemoryFree(pLogStore->data); + taosMemoryFree(pLogStore); } } @@ -48,18 +48,22 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(pEntry->index == logStoreLastIndex(pLogStore) + 1); + SyncIndex lastIndex = logStoreLastIndex(pLogStore); + assert(pEntry->index == lastIndex + 1); uint32_t len; char* serialized = syncEntrySerialize(pEntry, &len); assert(serialized != NULL); - int code; - code = walWrite(pWal, pEntry->index, pEntry->msgType, serialized, len); - assert(code == 0); + int code = 0; + /* + code = walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len); + assert(code == 0); + */ + assert(walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len) == 0); walFsync(pWal, true); - free(serialized); - return code; // to avoid compiler error + taosMemoryFree(serialized); + return code; } SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { @@ -69,7 +73,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - walReadWithHandle(pWalHandle, index); + assert(walReadWithHandle(pWalHandle, index) == 0); pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len); assert(pEntry != NULL); @@ -83,7 +87,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - walRollback(pWal, fromIndex); + assert(walRollback(pWal, fromIndex) == 0); return 0; // to avoid compiler error } @@ -99,7 +103,7 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { SSyncRaftEntry* pLastEntry = logStoreGetLastEntry(pLogStore); if (pLastEntry != NULL) { lastTerm = pLastEntry->term; - free(pLastEntry); + taosMemoryFree(pLastEntry); } return lastTerm; } @@ -107,7 +111,7 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - walCommit(pWal, index); + assert(walCommit(pWal, index) == 0); return 0; // to avoid compiler error } @@ -165,29 +169,84 @@ char* logStore2Str(SSyncLogStore* pLogStore) { return serialized; } +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { + char u64buf[128]; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); + return pJson; +} + +char* logStoreSimple2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStoreSimple2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + // for debug ----------------- void logStorePrint(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); printf("logStorePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void logStorePrint2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); printf("logStorePrint | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void logStoreLog(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); sTrace("logStorePrint | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void logStoreLog2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); sTrace("logStorePrint | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } + +// for debug ----------------- +void logStoreSimplePrint(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 3f6db129ce46de97b14da2b240b56878826f5e0e..c9054d088e1536eeb7c7f7208ab230231eb7c93c 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -26,7 +26,7 @@ static bool raftStoreFileExist(char *path); SRaftStore *raftStoreOpen(const char *path) { int32_t ret; - SRaftStore *pRaftStore = malloc(sizeof(SRaftStore)); + SRaftStore *pRaftStore = taosMemoryMalloc(sizeof(SRaftStore)); if (pRaftStore == NULL) { sError("raftStoreOpen malloc error"); return NULL; @@ -75,7 +75,7 @@ int32_t raftStoreClose(SRaftStore *pRaftStore) { assert(pRaftStore != NULL); taosCloseFile(&pRaftStore->pFile); - free(pRaftStore); + taosMemoryFree(pRaftStore); pRaftStore = NULL; return 0; } @@ -128,7 +128,7 @@ int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { assert(len2 < len); memset(buf, 0, len); snprintf(buf, len, "%s", serialized); - free(serialized); + taosMemoryFree(serialized); cJSON_Delete(pRoot); return 0; @@ -226,23 +226,23 @@ void raftStorePrint(SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); printf("raftStorePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void raftStorePrint2(char *s, SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); printf("raftStorePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void raftStoreLog(SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); sTrace("raftStoreLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void raftStoreLog2(char *s, SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); sTrace("raftStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 0bb701fc09a112234622a3f32830ba12e47445b7..3ced7a18bc53f274f48a2e936ab2a51a45227570 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -49,6 +49,10 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + syncIndexMgrLog2("==syncNodeAppendEntriesPeers== pNextIndex", pSyncNode->pNextIndex); + syncIndexMgrLog2("==syncNodeAppendEntriesPeers== pMatchIndex", pSyncNode->pMatchIndex); + logStoreSimpleLog2("==syncNodeAppendEntriesPeers==", pSyncNode->pLogStore); + int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId* pDestId = &(pSyncNode->peersId[i]); @@ -82,7 +86,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { assert(len == pEntry->bytes); memcpy(pMsg->data, serialized, len); - free(serialized); + taosMemoryFree(serialized); syncEntryDestory(pEntry); } else { @@ -99,6 +103,8 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { pMsg->prevLogTerm = preLogTerm; pMsg->commitIndex = pSyncNode->commitIndex; + syncAppendEntriesLog2("==syncNodeAppendEntriesPeers==", pMsg); + // send AppendEntries syncNodeAppendEntries(pSyncNode, pDestId, pMsg); syncAppendEntriesDestroy(pMsg); diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 062b0244bdd54ce611d7323d5e2115380f334733..e23748a81e7e8f8622c31d9bf98b381727a44c39 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -43,7 +43,10 @@ // int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; - syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteLog2(logBuf, pMsg); if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 12603bb337ce6791599bd6755756ff3ef2e090a7..1531f701ff7605543c6c5abaabcbb1d88d894528 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -38,7 +38,10 @@ // int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; - syncRequestVoteReplyLog2("==syncNodeOnRequestVoteReplyCb==", pMsg); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteReplyLog2(logBuf, pMsg); if (pMsg->term < ths->pRaftStore->currentTerm) { sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 9486405331bd3d5352eb8b1ce73cbff950b3feaa..287500a587feca5e0f5489de2faa2c80def38aad 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -76,10 +76,10 @@ bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId = // ---- SSyncBuffer ----- void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len) { syncBuf->len = len; - syncBuf->data = malloc(syncBuf->len); + syncBuf->data = taosMemoryMalloc(syncBuf->len); } -void syncUtilbufDestroy(SSyncBuffer* syncBuf) { free(syncBuf->data); } +void syncUtilbufDestroy(SSyncBuffer* syncBuf) { taosMemoryFree(syncBuf->data); } void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest) { dest->len = src->len; @@ -88,7 +88,7 @@ void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest) { void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { dest->len = src->len; - dest->data = malloc(dest->len); + dest->data = taosMemoryMalloc(dest->len); memcpy(dest->data, src->data, dest->len); } @@ -96,7 +96,10 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { int32_t syncUtilRand(int32_t max) { return taosRand() % max; } -int32_t syncUtilElectRandomMS() { return ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } +int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { + assert(min > 0 && max > 0 && max >= min); + return min + syncUtilRand(max - min); +} int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } @@ -158,7 +161,7 @@ bool syncUtilCanPrint(char c) { } char* syncUtilprintBin(char* ptr, uint32_t len) { - char* s = malloc(len + 1); + char* s = taosMemoryMalloc(len + 1); assert(s != NULL); memset(s, 0, len + 1); memcpy(s, ptr, len); @@ -173,7 +176,7 @@ char* syncUtilprintBin(char* ptr, uint32_t len) { char* syncUtilprintBin2(char* ptr, uint32_t len) { uint32_t len2 = len * 4 + 1; - char* s = malloc(len2); + char* s = taosMemoryMalloc(len2); assert(s != NULL); memset(s, 0, len2); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 69a4c48f39f9b532b106a3ba4b55871361b19a38..733dfd05b6deb88ed08df78858f358822bebbda7 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -23,7 +23,7 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { } SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { - SVotesGranted *pVotesGranted = malloc(sizeof(SVotesGranted)); + SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); assert(pVotesGranted != NULL); memset(pVotesGranted, 0, sizeof(SVotesGranted)); @@ -41,7 +41,7 @@ SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { void voteGrantedDestroy(SVotesGranted *pVotesGranted) { if (pVotesGranted != NULL) { - free(pVotesGranted); + taosMemoryFree(pVotesGranted); } } @@ -89,12 +89,12 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { for (int i = 0; i < pVotesGranted->replicaNum; ++i) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); } - int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pVotesGranted->replicaNum); for (int i = 0; i < pVotesGranted->replicaNum; ++i) { arr[i] = pVotesGranted->isGranted[i]; } cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); @@ -126,31 +126,31 @@ void voteGrantedPrint(SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); printf("voteGrantedPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedPrint2(char *s, SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); printf("voteGrantedPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedLog(SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); sTrace("voteGrantedLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedLog2(char *s, SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); sTrace("voteGrantedLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // SVotesRespond ----------------------------- SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { - SVotesRespond *pVotesRespond = malloc(sizeof(SVotesRespond)); + SVotesRespond *pVotesRespond = taosMemoryMalloc(sizeof(SVotesRespond)); assert(pVotesRespond != NULL); memset(pVotesRespond, 0, sizeof(SVotesRespond)); @@ -164,7 +164,7 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { void votesRespondDestory(SVotesRespond *pVotesRespond) { if (pVotesRespond != NULL) { - free(pVotesRespond); + taosMemoryFree(pVotesRespond); } } @@ -213,7 +213,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); } int respondNum = 0; - int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pVotesRespond->replicaNum); for (int i = 0; i < pVotesRespond->replicaNum; ++i) { arr[i] = pVotesRespond->isRespond[i]; if (pVotesRespond->isRespond[i]) { @@ -221,7 +221,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { } } cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); @@ -248,24 +248,24 @@ void votesRespondPrint(SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); printf("votesRespondPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void votesRespondPrint2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); printf("votesRespondPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void votesRespondLog(SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); sTrace("votesRespondLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void votesRespondLog2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); sTrace("votesRespondLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 0c7608f902621ba8b233f67a6844fdf953b360fa..7341565b1d526239e2b15aae2d463b294991c3da 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -28,9 +28,16 @@ add_executable(syncRpcMsgTest "") add_executable(syncPingTimerTest2 "") add_executable(syncPingSelfTest "") add_executable(syncElectTest "") +add_executable(syncElectTest2 "") +add_executable(syncElectTest3 "") add_executable(syncEncodeTest "") add_executable(syncWriteTest "") add_executable(syncReplicateTest "") +add_executable(syncReplicateTest2 "") +add_executable(syncReplicateTest3 "") +add_executable(syncReplicateLoadTest "") +add_executable(syncRefTest "") +add_executable(syncLogStoreCheck "") target_sources(syncTest @@ -153,6 +160,14 @@ target_sources(syncElectTest PRIVATE "syncElectTest.cpp" ) +target_sources(syncElectTest2 + PRIVATE + "syncElectTest2.cpp" +) +target_sources(syncElectTest3 + PRIVATE + "syncElectTest3.cpp" +) target_sources(syncEncodeTest PRIVATE "syncEncodeTest.cpp" @@ -165,6 +180,26 @@ target_sources(syncReplicateTest PRIVATE "syncReplicateTest.cpp" ) +target_sources(syncReplicateTest2 + PRIVATE + "syncReplicateTest2.cpp" +) +target_sources(syncReplicateTest3 + PRIVATE + "syncReplicateTest3.cpp" +) +target_sources(syncReplicateLoadTest + PRIVATE + "syncReplicateLoadTest.cpp" +) +target_sources(syncRefTest + PRIVATE + "syncRefTest.cpp" +) +target_sources(syncLogStoreCheck + PRIVATE + "syncLogStoreCheck.cpp" +) target_include_directories(syncTest @@ -317,7 +352,12 @@ target_include_directories(syncElectTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncElectTest +target_include_directories(syncElectTest2 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncElectTest3 PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" @@ -337,6 +377,31 @@ target_include_directories(syncReplicateTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncReplicateTest2 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncReplicateTest3 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncReplicateLoadTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncRefTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncLogStoreCheck + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -459,6 +524,14 @@ target_link_libraries(syncElectTest sync gtest_main ) +target_link_libraries(syncElectTest2 + sync + gtest_main +) +target_link_libraries(syncElectTest3 + sync + gtest_main +) target_link_libraries(syncEncodeTest sync gtest_main @@ -471,6 +544,26 @@ target_link_libraries(syncReplicateTest sync gtest_main ) +target_link_libraries(syncReplicateTest2 + sync + gtest_main +) +target_link_libraries(syncReplicateTest3 + sync + gtest_main +) +target_link_libraries(syncReplicateLoadTest + sync + gtest_main +) +target_link_libraries(syncRefTest + sync + gtest_main +) +target_link_libraries(syncLogStoreCheck + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index 362da67c66869a85a06b6d34398ffd5f30d0f165..72aeb155e7eb0a0aec376802e57849b1410a3ccc 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -34,14 +34,14 @@ void test1() { void test2() { SyncAppendEntriesReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncAppendEntriesReplySerialize(pMsg, serialized, len); SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(); syncAppendEntriesReplyDeserialize(serialized, len, pMsg2); syncAppendEntriesReplyPrint2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesReplyDestroy(pMsg); syncAppendEntriesReplyDestroy(pMsg2); } @@ -54,7 +54,7 @@ void test3() { syncAppendEntriesReplyPrint2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesReplyDestroy(pMsg); syncAppendEntriesReplyDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index 687d9bcb945dbbcddcf0d543f67ea578c5bc4617..69a0aee9a8b389a322669adee70459d07e7c3f11 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -36,13 +36,13 @@ void test1() { void test2() { SyncAppendEntries *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncAppendEntriesSerialize(pMsg, serialized, len); SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen); syncAppendEntriesDeserialize(serialized, len, pMsg2); syncAppendEntriesPrint2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesDestroy(pMsg); syncAppendEntriesDestroy(pMsg2); } @@ -54,7 +54,7 @@ void test3() { SyncAppendEntries *pMsg2 = syncAppendEntriesDeserialize2(serialized, len); syncAppendEntriesPrint2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesDestroy(pMsg); syncAppendEntriesDestroy(pMsg2); } @@ -63,7 +63,7 @@ void test4() { SyncAppendEntries *pMsg = createMsg(); SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); - SyncAppendEntries *pMsg2 = (SyncAppendEntries *)malloc(rpcMsg.contLen); + SyncAppendEntries *pMsg2 = (SyncAppendEntries *)taosMemoryMalloc(rpcMsg.contLen); syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2); syncAppendEntriesPrint2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncClientRequestTest.cpp b/source/libs/sync/test/syncClientRequestTest.cpp index 6323f53a0354897e5b2f6525573e4eeade592014..f22478d53878f88cab0461f5af619f8f20070b51 100644 --- a/source/libs/sync/test/syncClientRequestTest.cpp +++ b/source/libs/sync/test/syncClientRequestTest.cpp @@ -34,13 +34,13 @@ void test1() { void test2() { SyncClientRequest *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncClientRequestSerialize(pMsg, serialized, len); SyncClientRequest *pMsg2 = syncClientRequestBuild(pMsg->dataLen); syncClientRequestDeserialize(serialized, len, pMsg2); syncClientRequestPrint2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncClientRequestDestroy(pMsg); syncClientRequestDestroy(pMsg2); } @@ -52,7 +52,7 @@ void test3() { SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); syncClientRequestPrint2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncClientRequestDestroy(pMsg); syncClientRequestDestroy(pMsg2); } @@ -61,7 +61,7 @@ void test4() { SyncClientRequest *pMsg = createMsg(); SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)malloc(rpcMsg.contLen); + SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); syncClientRequestPrint2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 502263cfbf736758298961211db6eda5876de2a3..251a4b538fe24574325136cb36f6f4037bdd8078 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -88,7 +88,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -116,9 +116,10 @@ int main(int argc, char** argv) { //--------------------------- while (1) { - sTrace("while 1 sleep, state: %d, %s, electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", - gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->electTimerLogicClock, - gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + sTrace( + "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, + gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncElectTest2.cpp b/source/libs/sync/test/syncElectTest2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..709d9d8580f91c6221456019a9c6a67a8eeb8736 --- /dev/null +++ b/source/libs/sync/test/syncElectTest2.cpp @@ -0,0 +1,135 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "tref.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./elect2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"", pSyncNode); + initRaftId(pSyncNode); + + //--------------------------- + while (1) { + sTrace( + "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + + return 0; +} diff --git a/source/libs/sync/test/syncElectTest3.cpp b/source/libs/sync/test/syncElectTest3.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e36a2d0fb1c14669fe10d90ca0ace4aa8e877a9f --- /dev/null +++ b/source/libs/sync/test/syncElectTest3.cpp @@ -0,0 +1,138 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "tref.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect3_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./elect3_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + pSyncNode->hbBaseLine = 500; + pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"", pSyncNode); + initRaftId(pSyncNode); + + //--------------------------- + while (1) { + sTrace( + "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + + return 0; +} diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index e554858072bf6a3f71c8ec1c185ead2274a0b852..b1ba95dc3e46d5ecfe08afa007bba860cb33eab2 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -87,16 +87,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0() { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 32; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); return pMsg; } @@ -107,7 +107,7 @@ SyncClientRequest *step1(const SRpcMsg *pMsg) { } SRpcMsg *step2(const SyncClientRequest *pMsg) { - SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pRetMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); syncClientRequest2RpcMsg(pMsg, pRetMsg); return pRetMsg; } @@ -133,7 +133,7 @@ SSyncRaftEntry *step6(const char *pMsg, uint32_t len) { } SRpcMsg *step7(const SSyncRaftEntry *pMsg) { - SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pRetMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); syncEntry2OriginalRpc(pMsg, pRetMsg); return pRetMsg; } @@ -190,7 +190,7 @@ int main(int argc, char **argv) { char * pMsg5 = step5(pMsg4, &len); char * s = syncUtilprintBin(pMsg5, len); printf("==step5== [%s] \n", s); - free(s); + taosMemoryFree(s); // step6 SSyncRaftEntry *pMsg6 = step6(pMsg5, len); diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index 57315f40ec31a9148727ac1c4a822c267b119b36..48cc7caf2f433d3268c8a6b8e7b2ad44231f0d0a 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -66,7 +66,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index e54daaa79a052504fd8cb8bbbb7f59baec9e6385..e8427e81685a22e293fab61ac34071adb7b3b54f 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -46,6 +46,20 @@ void test2() { } void test3() { + SyncClientRequest* pSyncMsg = syncClientRequestBuild(10); + pSyncMsg->originalRpcType = 33; + pSyncMsg->seqNum = 11; + pSyncMsg->isWeak = 1; + strcpy(pSyncMsg->data, "test3"); + + SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200, SYNC_RAFT_ENTRY_NOOP); + syncEntryPrint(pEntry); + + syncClientRequestDestroy(pSyncMsg); + syncEntryDestory(pEntry); +} + +void test4() { SSyncRaftEntry* pEntry = syncEntryBuild(10); assert(pEntry != NULL); pEntry->msgType = 11; @@ -54,7 +68,8 @@ void test3() { pEntry->isWeak = true; pEntry->term = 44; pEntry->index = 55; - strcpy(pEntry->data, "test3"); + pEntry->entryType = SYNC_RAFT_ENTRY_CONFIG; + strcpy(pEntry->data, "test4"); syncEntryPrint(pEntry); uint32_t len; @@ -63,7 +78,7 @@ void test3() { SSyncRaftEntry* pEntry2 = syncEntryDeserialize(serialized, len); syncEntryPrint(pEntry2); - free(serialized); + taosMemoryFree(serialized); syncEntryDestory(pEntry2); syncEntryDestory(pEntry); } @@ -76,6 +91,7 @@ int main(int argc, char** argv) { test1(); test2(); test3(); + test4(); return 0; } diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index 0fc3ebfe4cdc5e4417f34c905c039ac5c31fc707..913e57e59abfcf7ded83c9061d33026243dea2fb 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -66,7 +66,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 6bad8f09cf3f5912f0ef4ba71316dd3654af91fd..319ea3e15a176f5f77df0cc88fbd8d953d7678f4 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -68,7 +68,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -93,7 +93,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -105,7 +105,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); @@ -117,7 +117,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } printf("---------------------------------------\n"); @@ -132,7 +132,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } syncIndexMgrDestroy(pSyncIndexMgr); diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index a3e5f41c85765e771cc8ab36efd663117f4cecab..48b5488e416c442ffacff6e0884efefabffbb6ec 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp new file mode 100644 index 0000000000000000000000000000000000000000..85e39a61c56c47f7f90d6465c5f70f5a508f5ea8 --- /dev/null +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -0,0 +1,105 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit(const char* path) { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./log_check"); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* logStoreCheck(const char* path) { return syncNodeInit(path); } + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + pSyncNode = logStoreCheck(argv[1]); + assert(pSyncNode != NULL); + + logStorePrint2((char*)"logStoreCheck", pSyncNode->pLogStore); + + return 0; +} diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index c1cb66f382574f042bf910308a5da5e2d1a1b0b1..a5adba9a8861b7f12f8bc6259e7f466c2b3b27db 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -116,7 +116,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingReplyTest.cpp b/source/libs/sync/test/syncPingReplyTest.cpp index 8e1448e7818c8d41c6b880208c2f4fe78ecb4035..40592eab6f7935fb048c27455c1aa4aace077c17 100644 --- a/source/libs/sync/test/syncPingReplyTest.cpp +++ b/source/libs/sync/test/syncPingReplyTest.cpp @@ -33,13 +33,13 @@ void test1() { void test2() { SyncPingReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncPingReplySerialize(pMsg, serialized, len); SyncPingReply *pMsg2 = syncPingReplyBuild(pMsg->dataLen); syncPingReplyDeserialize(serialized, len, pMsg2); syncPingReplyPrint2((char *)"test2: syncPingReplySerialize -> syncPingReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } @@ -51,7 +51,7 @@ void test3() { SyncPingReply *pMsg2 = syncPingReplyDeserialize2(serialized, len); syncPingReplyPrint2((char *)"test3: syncPingReplySerialize3 -> syncPingReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } @@ -60,7 +60,7 @@ void test4() { SyncPingReply *pMsg = createMsg(); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); - SyncPingReply *pMsg2 = (SyncPingReply *)malloc(rpcMsg.contLen); + SyncPingReply *pMsg2 = (SyncPingReply *)taosMemoryMalloc(rpcMsg.contLen); syncPingReplyFromRpcMsg(&rpcMsg, pMsg2); syncPingReplyPrint2((char *)"test4: syncPingReply2RpcMsg -> syncPingReplyFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 05e4d99cb0cb44388f82bf154ff28769d5766162..677518c6ac76b3d16f80bfdefb4361655f245ba4 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 83394b0e77404b32ef665643fc85cdf6f8a602b6..eb774d77c31166071079ad4b39fc30f64743fce3 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -33,13 +33,13 @@ void test1() { void test2() { SyncPing *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncPingSerialize(pMsg, serialized, len); SyncPing *pMsg2 = syncPingBuild(pMsg->dataLen); syncPingDeserialize(serialized, len, pMsg2); syncPingPrint2((char *)"test2: syncPingSerialize -> syncPingDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingDestroy(pMsg); syncPingDestroy(pMsg2); } @@ -51,7 +51,7 @@ void test3() { SyncPing *pMsg2 = syncPingDeserialize2(serialized, len); syncPingPrint2((char *)"test3: syncPingSerialize3 -> syncPingDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingDestroy(pMsg); syncPingDestroy(pMsg2); } @@ -60,7 +60,7 @@ void test4() { SyncPing *pMsg = createMsg(); SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); - SyncPing *pMsg2 = (SyncPing *)malloc(rpcMsg.contLen); + SyncPing *pMsg2 = (SyncPing *)taosMemoryMalloc(rpcMsg.contLen); syncPingFromRpcMsg(&rpcMsg, pMsg2); syncPingPrint2((char *)"test4: syncPing2RpcMsg -> syncPingFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index 20d4a9ce589bad5b399f727729a9fe7f21c0d94f..3c5e76ca147aeb2b438bc2a5afb0e90417cf0cc9 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index 2a041f3f5ddf4a530127699a0e64645bfd40b1f4..554f67d3658480c34793e3c3663d7caf84142688 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 688802625ac42203b7f52a459b7440ff3324f8e8..460ff90f4f752b76182fc635fe62505ca2e37f66 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -25,7 +25,7 @@ void initRaftId() { ids[i].vgId = 1234; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncRefTest.cpp b/source/libs/sync/test/syncRefTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..96062b1a91b235ce24a11cfb91a0b13ac14c5c23 --- /dev/null +++ b/source/libs/sync/test/syncRefTest.cpp @@ -0,0 +1,135 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "tref.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +static void syncFreeObj(void *param); +int32_t init(); +void cleanup(); +int64_t start(); +void stop(int64_t rid); + +static int32_t tsNodeRefId = -1; +int g = 100; + +typedef struct SyncObj { + int64_t rid; + void * data; + char name[32]; + int counter; +} SyncObj; + +static void syncFreeObj(void *param) { + SyncObj *pObj = (SyncObj *)param; + printf("syncFreeObj name:%s rid:%ld \n", pObj->name, pObj->rid); + taosMemoryFree(pObj); +} + +int32_t init() { + tsNodeRefId = taosOpenRef(200, syncFreeObj); + if (tsNodeRefId < 0) { + sError("failed to init node ref"); + cleanup(); + return -1; + } + return 0; +} + +void cleanup() { + if (tsNodeRefId != -1) { + taosCloseRef(tsNodeRefId); + tsNodeRefId = -1; + } +} + +int64_t start() { + SyncObj *pObj = (SyncObj *)taosMemoryMalloc(sizeof(SyncObj)); + assert(pObj != NULL); + + pObj->data = &g; + snprintf(pObj->name, sizeof(pObj->name), "%s", "hello"); + + pObj->rid = taosAddRef(tsNodeRefId, pObj); + if (pObj->rid < 0) { + syncFreeObj(pObj); + return -1; + } + + printf("start name:%s rid:%ld \n", pObj->name, pObj->rid); + return pObj->rid; +} + +void stop(int64_t rid) { + SyncObj *pObj = (SyncObj *)taosAcquireRef(tsNodeRefId, rid); + if (pObj == NULL) return; + + printf("stop name:%s rid:%ld \n", pObj->name, pObj->rid); + pObj->data = NULL; + + taosReleaseRef(tsNodeRefId, pObj->rid); + taosRemoveRef(tsNodeRefId, rid); +} + +void *func(void *param) { + int64_t rid = (int64_t)param; + + int32_t ms = taosRand() % 10000; + taosMsleep(ms); + + SyncObj *pObj = (SyncObj *)taosAcquireRef(tsNodeRefId, rid); + if (pObj != NULL) { + printf("taosAcquireRef sleep:%d, name:%s, rid:%ld \n", ms, pObj->name, pObj->rid); + } else { + printf("taosAcquireRef sleep:%d, NULL! \n", ms); + } + + taosReleaseRef(tsNodeRefId, rid); + return NULL; +} + +int main() { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + logTest(); + + taosSeedRand(taosGetTimestampSec()); + + int32_t ret; + + ret = init(); + assert(ret == 0); + + int64_t rid = start(); + assert(rid > 0); + + for (int i = 0; i < 20; ++i) { + TdThread tid; + taosThreadCreate(&tid, NULL, func, (void *)rid); + } + + int32_t ms = taosRand() % 10000; + taosMsleep(ms); + printf("main sleep %d, stop and clean ", ms); + + stop(rid); + cleanup(); + + while (1) { + taosMsleep(1000); + printf("sleep 1 ... \n"); + } + + return 0; +} diff --git a/source/libs/sync/test/syncReplicateLoadTest.cpp b/source/libs/sync/test/syncReplicateLoadTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0841083b4a5cb00db2f4f0b3cc758c991fd72364 --- /dev/null +++ b/source/libs/sync/test/syncReplicateLoadTest.cpp @@ -0,0 +1,189 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + // only load ... + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 502263cfbf736758298961211db6eda5876de2a3..9783bac7e551076bc71a33194af24de530883c6b 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -3,6 +3,8 @@ #include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" @@ -22,18 +24,50 @@ int32_t myIndex = 0; SRaftId ids[TSDB_MAX_REPLICA]; SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; -SSyncNode* gSyncNode; +SSyncFSM * pFsm; +SWal * pWal; +SSyncNode *gSyncNode; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} -SSyncNode* syncNodeInit() { +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +SSyncNode *syncNodeInit() { syncInfo.vgId = 1234; syncInfo.rpcClient = gSyncIO->clientRpc; syncInfo.FpSendMsg = syncIOSendMsg; syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect_test_%d", myIndex); + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate_test_%d", myIndex); int code = walInit(); assert(code == 0); @@ -48,13 +82,13 @@ SSyncNode* syncNodeInit() { walCfg.level = TAOS_WAL_FSYNC; char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./elect_test_wal_%d", myIndex); + snprintf(tmpdir, sizeof(tmpdir), "./replicate_test_wal_%d", myIndex); pWal = walOpen(tmpdir, &walCfg); assert(pWal != NULL); syncInfo.pWal = pWal; - SSyncCfg* pCfg = &syncInfo.syncCfg; + SSyncCfg *pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; pCfg->replicaNum = replicaNum; @@ -64,61 +98,105 @@ SSyncNode* syncNodeInit() { // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } - SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + SSyncNode *pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; return pSyncNode; } -SSyncNode* syncInitTest() { return syncNodeInit(); } +SSyncNode *syncInitTest() { return syncNodeInit(); } -void initRaftId(SSyncNode* pSyncNode) { +void initRaftId(SSyncNode *pSyncNode) { for (int i = 0; i < replicaNum; ++i) { ids[i] = pSyncNode->replicasId[i]; - char* s = syncUtilRaftId2Str(&ids[i]); + char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } -int main(int argc, char** argv) { +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; + void logTest(); myIndex = 0; if (argc >= 2) { myIndex = atoi(argv[1]); } - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); assert(ret == 0); ret = syncEnvStart(); assert(ret == 0); + taosRemoveDir("./wal_test"); + + initFsm(); + gSyncNode = syncInitTest(); assert(gSyncNode != NULL); - syncNodePrint2((char*)"", gSyncNode); + syncNodePrint2((char *)"", gSyncNode); initRaftId(gSyncNode); - //--------------------------- + for (int i = 0; i < 30; ++i) { + // step0 + SRpcMsg *pMsg0 = step0(i); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + // step1 + SyncClientRequest *pMsg1 = step1(pMsg0); + syncClientRequestPrint2((char *)"==step1==", pMsg1); + + SyncClientRequest *pSyncClientRequest = pMsg1; + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + gSyncNode->FpEqMsg(gSyncNode->queue, &rpcMsg); + + taosMsleep(1000); + sTrace( + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, + gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS, + gSyncNode->commitIndex); + } + while (1) { - sTrace("while 1 sleep, state: %d, %s, electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", - gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->electTimerLogicClock, - gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, + gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS, + gSyncNode->commitIndex); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncReplicateTest2.cpp b/source/libs/sync/test/syncReplicateTest2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..430df7eebdbc7333d9240bdc4e4a8d13ccb18937 --- /dev/null +++ b/source/libs/sync/test/syncReplicateTest2.cpp @@ -0,0 +1,205 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + for (int i = 0; i < 30; ++i) { + // step0 + SRpcMsg *pMsg0 = step0(i); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + syncPropose(rid, pMsg0, true); + taosMsleep(1000); + + taosMemoryFree(pMsg0); + + sTrace( + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + } + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncReplicateTest3.cpp b/source/libs/sync/test/syncReplicateTest3.cpp new file mode 100644 index 0000000000000000000000000000000000000000..792b9c94cf022583e2cbee5185e8ef4dd7eefef4 --- /dev/null +++ b/source/libs/sync/test/syncReplicateTest3.cpp @@ -0,0 +1,217 @@ +#define ALLOW_FORBID_FUNC + +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int recordCount = 100; + if (argc >= 3) { + recordCount = atoi(argv[2]); + } + + int sleepMS = 10; + if (argc >= 4) { + sleepMS = atoi(argv[3]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + for (int i = 0; i < recordCount; ++i) { + // step0 + SRpcMsg *pMsg0 = step0(i); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + syncPropose(rid, pMsg0, true); + taosMsleep(sleepMS); + + taosMemoryFree(pMsg0); + + sTrace( + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + } + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncRequestVoteReplyTest.cpp b/source/libs/sync/test/syncRequestVoteReplyTest.cpp index 2bce3e4cd6b00e077e6e5abc54f54a94b357995e..9e0ebee31318be2baa71e3b927dc2adb4269260f 100644 --- a/source/libs/sync/test/syncRequestVoteReplyTest.cpp +++ b/source/libs/sync/test/syncRequestVoteReplyTest.cpp @@ -34,13 +34,13 @@ void test1() { void test2() { SyncRequestVoteReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncRequestVoteReplySerialize(pMsg, serialized, len); SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyBuild(); syncRequestVoteReplyDeserialize(serialized, len, pMsg2); syncRequestVoteReplyPrint2((char *)"test2: syncRequestVoteReplySerialize -> syncRequestVoteReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteReplyDestroy(pMsg); syncRequestVoteReplyDestroy(pMsg2); } @@ -53,7 +53,7 @@ void test3() { syncRequestVoteReplyPrint2((char *)"test3: syncRequestVoteReplySerialize3 -> syncRequestVoteReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteReplyDestroy(pMsg); syncRequestVoteReplyDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 22f47046dea3a8d98e00e5b3fa28d89dfdfd7a04..6b2120e124f72dd64d32c9e6ed3e2b8edabd9df1 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -35,13 +35,13 @@ void test1() { void test2() { SyncRequestVote *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncRequestVoteSerialize(pMsg, serialized, len); SyncRequestVote *pMsg2 = syncRequestVoteBuild(); syncRequestVoteDeserialize(serialized, len, pMsg2); syncRequestVotePrint2((char *)"test2: syncRequestVoteSerialize -> syncRequestVoteDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteDestroy(pMsg); syncRequestVoteDestroy(pMsg2); } @@ -53,7 +53,7 @@ void test3() { SyncRequestVote *pMsg2 = syncRequestVoteDeserialize2(serialized, len); syncRequestVotePrint2((char *)"test3: syncRequestVoteSerialize3 -> syncRequestVoteDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteDestroy(pMsg); syncRequestVoteDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index 3f46ab5c7cebc4283ba8edfcbc76315d1a360023..ab36f42d0d00457cb9d6ca602d2623b9c1296971 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -30,13 +30,13 @@ void test1() { void test2() { SyncTimeout *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncTimeoutSerialize(pMsg, serialized, len); SyncTimeout *pMsg2 = syncTimeoutBuild(); syncTimeoutDeserialize(serialized, len, pMsg2); syncTimeoutPrint2((char *)"test2: syncTimeoutSerialize -> syncTimeoutDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncTimeoutDestroy(pMsg); syncTimeoutDestroy(pMsg2); } @@ -48,7 +48,7 @@ void test3() { SyncTimeout *pMsg2 = syncTimeoutDeserialize2(serialized, len); syncTimeoutPrint2((char *)"test3: syncTimeoutSerialize3 -> syncTimeoutDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncTimeoutDestroy(pMsg); syncTimeoutDestroy(pMsg2); } @@ -57,7 +57,7 @@ void test4() { SyncTimeout *pMsg = createMsg(); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pMsg, &rpcMsg); - SyncTimeout *pMsg2 = (SyncTimeout *)malloc(rpcMsg.contLen); + SyncTimeout *pMsg2 = (SyncTimeout *)taosMemoryMalloc(rpcMsg.contLen); syncTimeoutFromRpcMsg(&rpcMsg, pMsg2); syncTimeoutPrint2((char *)"test4: syncTimeout2RpcMsg -> syncTimeoutFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index 663db3a7b301eb8e020ab5f2f95066ca06d8a308..ae2afd2f536458ef26b5fe7731ba06e403f4af5f 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -16,7 +16,7 @@ void logTest() { void electRandomMSTest() { for (int i = 0; i < 10; ++i) { - int32_t ms = syncUtilElectRandomMS(); + int32_t ms = syncUtilElectRandomMS(150, 300); printf("syncUtilElectRandomMS: %d \n", ms); } } diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 588eb32ffd75d648879f923e619cd624aab5ba68..6164aed3513128daf41bcade0e68c4f1c1e94956 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -67,7 +67,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -92,7 +92,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -104,7 +104,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } SyncTerm term = 1234; @@ -114,7 +114,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } for (int i = 0; i < replicaNum; ++i) { @@ -129,7 +129,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } voteGrantedVote(pVotesGranted, reply); @@ -137,7 +137,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } } @@ -147,7 +147,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } voteGrantedDestroy(pVotesGranted); diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index 76fd6fab4e1d807565d3f79f0d0910b6f8111953..39ba6253a1a61974623939ed055dc51247d7eab3 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -67,7 +67,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -92,7 +92,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -104,7 +104,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } SyncTerm term = 1234; @@ -114,7 +114,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } for (int i = 0; i < replicaNum; ++i) { @@ -129,7 +129,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } votesRespondAdd(pVotesRespond, reply); @@ -137,7 +137,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } } @@ -147,7 +147,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } votesRespondDestory(pVotesRespond); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 2598abbddd86ac6813afd287246202d5439cb9c8..a3c24f69ff7d5981cd8ad19f97bb9e9fa7866eec 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -28,23 +28,33 @@ SSyncFSM * pFsm; SWal * pWal; SSyncNode *gSyncNode; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf); +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf); +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf); +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { - pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -108,16 +118,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0() { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 32; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); return pMsg; } diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index f16d0445c6bb2ab9eb5aaf6e5d7844527637b538..2a508cf676f744d0ffd09e4b656fcb15df63c82a 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -41,7 +41,7 @@ typedef struct { } STfsDisk; typedef struct { - pthread_spinlock_t lock; + TdThreadSpinlock lock; int32_t level; int32_t nextid; // next disk id to allocate int32_t ndisk; // # of disks mounted to this tier @@ -64,7 +64,7 @@ typedef struct STfsDir { } STfsDir; typedef struct STfs { - pthread_spinlock_t lock; + TdThreadSpinlock lock; SDiskSize size; int32_t nlevel; STfsTier tiers[TFS_MAX_TIERS]; @@ -82,11 +82,11 @@ void tfsUpdateTierSize(STfsTier *pTier); int32_t tfsAllocDiskOnTier(STfsTier *pTier); void tfsPosNextId(STfsTier *pTier); -#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock) -#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) +#define tfsLockTier(pTier) taosThreadSpinLock(&(pTier)->lock) +#define tfsUnLockTier(pTier) taosThreadSpinUnlock(&(pTier)->lock) -#define tfsLock(pTfs) pthread_spin_lock(&(pTfs)->lock) -#define tfsUnLock(pTfs) pthread_spin_unlock(&(pTfs)->lock) +#define tfsLock(pTfs) taosThreadSpinLock(&(pTfs)->lock) +#define tfsUnLock(pTfs) taosThreadSpinUnlock(&(pTfs)->lock) #define TFS_TIER_AT(pTfs, level) (&(pTfs)->tiers[level]) #define TFS_DISK_AT(pTfs, did) ((pTfs)->tiers[(did).level].disks[(did).id]) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index aee7376491c9b7c395456ce7d1bc20559b70c969..e50045d30b41abb363d28f5958645829d2c893ee 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -30,13 +30,13 @@ STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { return NULL; } - STfs *pTfs = calloc(1, sizeof(STfs)); + STfs *pTfs = taosMemoryCalloc(1, sizeof(STfs)); if (pTfs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - if (pthread_spin_init(&pTfs->lock, 0) != 0) { + if (taosThreadSpinInit(&pTfs->lock, 0) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); tfsClose(pTfs); return NULL; @@ -85,8 +85,8 @@ void tfsClose(STfs *pTfs) { } taosHashCleanup(pTfs->hash); - pthread_spin_destroy(&pTfs->lock); - free(pTfs); + taosThreadSpinDestroy(&pTfs->lock); + taosMemoryFree(pTfs); } void tfsUpdateSize(STfs *pTfs) { @@ -184,7 +184,7 @@ void *tfsDecodeFile(STfs *pTfs, void *buf, STfsFile *pFile) { tfsInitFile(pTfs, pFile, diskId, rname); - tfree(rname); + taosMemoryFreeClear(rname); return buf; } @@ -242,12 +242,12 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { char *dir = strdup(taosDirName(s)); if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) { - free(s); - free(dir); + taosMemoryFree(s); + taosMemoryFree(dir); return -1; } - free(s); - free(dir); + taosMemoryFree(s); + taosMemoryFree(dir); if (tfsMkdirAt(pTfs, rname, diskId) < 0) { return -1; @@ -311,7 +311,7 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { } STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { - STfsDir *pDir = calloc(1, sizeof(STfsDir)); + STfsDir *pDir = taosMemoryCalloc(1, sizeof(STfsDir)); if (pDir == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -323,7 +323,7 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { tstrncpy(pDir->dirname, rname, TSDB_FILENAME_LEN); if (tfsOpendirImpl(pTfs, pDir) < 0) { - free(pDir); + taosMemoryFree(pDir); return NULL; } @@ -369,7 +369,7 @@ void tfsClosedir(STfsDir *pTfsDir) { taosCloseDir(pTfsDir->pDir); pTfsDir->pDir = NULL; } - free(pTfsDir); + taosMemoryFree(pTfsDir); } } @@ -395,7 +395,7 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { } static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { - char dirName[TSDB_FILENAME_LEN] = "\0"; + char dirName[TSDB_FILENAME_LEN] = "\0"; if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 52396db3be470bcb9735f10bef8789ec883089a3..ff40529ab2d75ec90be9b35ad259594f869d048e 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -17,7 +17,7 @@ #include "tfsInt.h" STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { - STfsDisk *pDisk = calloc(1, sizeof(STfsDisk)); + STfsDisk *pDisk = taosMemoryCalloc(1, sizeof(STfsDisk)); if (pDisk == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -25,7 +25,7 @@ STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { pDisk->path = strdup(path); if (pDisk->path == NULL) { - free(pDisk); + taosMemoryFree(pDisk); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -38,8 +38,8 @@ STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { STfsDisk *tfsFreeDisk(STfsDisk *pDisk) { if (pDisk != NULL) { - free(pDisk->path); - free(pDisk); + taosMemoryFree(pDisk->path); + taosMemoryFree(pDisk); } return NULL; diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index e4390d13d11a8b331d4e13a93e10f0ec5c1397c2..2ad021cbb543ff3f65ec9a558278451c69791d31 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -19,7 +19,7 @@ int32_t tfsInitTier(STfsTier *pTier, int32_t level) { memset(pTier, 0, sizeof(STfsTier)); - if (pthread_spin_init(&pTier->lock, 0) != 0) { + if (taosThreadSpinInit(&pTier->lock, 0) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -34,7 +34,7 @@ void tfsDestroyTier(STfsTier *pTier) { } pTier->ndisk = 0; - pthread_spin_destroy(&pTier->lock); + taosThreadSpinDestroy(&pTier->lock); } STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) { diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 9cb914a670e58ad6ca6f541a9677ed32dfd55925..1a093c3877a3c47e73af34cfc729887062d98e3a 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -196,7 +196,7 @@ TEST_F(TfsTest, 04_File) { { int32_t size = 1024; - void *ret = malloc(size + sizeof(size_t)); + void *ret = taosMemoryMalloc(size + sizeof(size_t)); *(size_t *)ret = size; void *buf = (void *)((char *)ret + sizeof(size_t)); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 8ea65b193d2911878bd9e74af8c814232c954a27..1b8cbb0f0407052e35294a990652286dd205627b 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -14,6 +14,10 @@ */ #ifdef USE_UV +#ifdef __cplusplus +extern "C" { +#endif + #include #include "lz4.h" #include "os.h" @@ -121,24 +125,21 @@ typedef struct { } SRpcReqContext; typedef SRpcMsg STransMsg; +typedef SRpcCtx STransCtx; +typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; typedef struct { - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app - int32_t contLen; // content length - // int32_t code; // error code - // int16_t numOfTry; // number of try for different servers - // int8_t oldInUse; // server EP inUse passed by app - // int8_t redirect; // flag to indicate redirect + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + tmsg_t msgType; // message type int8_t connType; // connection type cli/srv int64_t rid; // refId returned by taosAddRef - STransMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API + STransCtx appCtx; // + STransMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API int hThrdIdx; char* ip; @@ -150,13 +151,15 @@ typedef struct { typedef struct { char version : 4; // RPC version - char comp : 4; // compression algorithm, 0:no compression 1:lz4 - char resflag : 2; // reserved bits - char spi : 1; // security parameter index + char comp : 2; // compression algorithm, 0:no compression 1:lz4 + char noResp : 2; // noResp bits, 0: resp, 1: resp + char persist : 2; // persist handle,0: no persit, 1: persist handle + char release : 2; char secured : 2; - char encrypt : 3; // encrypt algorithm, 0: no encryption + char spi : 2; - uint32_t code; // del later + uint64_t ahandle; // ahandle assigned by client + uint32_t code; // del later uint32_t msgType; int32_t msgLen; uint8_t content[0]; // message body starts from here @@ -179,6 +182,9 @@ typedef struct { #pragma pack(pop) +typedef enum { Normal, Quit, Release, Register } STransMsgType; +typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } ConnStatus; + #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) #define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) @@ -223,9 +229,9 @@ typedef struct SConnBuffer { typedef void (*AsyncCB)(uv_async_t* handle); typedef struct { - void* pThrd; - queue qmsg; - pthread_mutex_t mtx; // protect qmsg; + void* pThrd; + queue qmsg; + TdThreadMutex mtx; // protect qmsg; } SAsyncItem; typedef struct { @@ -255,9 +261,10 @@ void transUnrefCliHandle(void* handle); void transReleaseCliHandle(void* handle); void transReleaseSrvHandle(void* handle); -void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg); +void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* pCtx); void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp); -void transSendResponse(const STransMsg* pMsg); +void transSendResponse(const STransMsg* msg); +void transRegisterMsg(const STransMsg* msg); int transGetConnInfo(void* thandle, STransHandleInfo* pInfo); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); @@ -266,4 +273,61 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void transCloseClient(void* arg); void transCloseServer(void* arg); +void transCtxInit(STransCtx* ctx); +void transCtxCleanup(STransCtx* ctx); +void transCtxClear(STransCtx* ctx); +void transCtxMerge(STransCtx* dst, STransCtx* src); +void* transCtxDumpVal(STransCtx* ctx, int32_t key); +void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType); + +// queue sending msgs +typedef struct { + SArray* q; + void (*freeFunc)(const void* arg); +} STransQueue; + +/* + * init queue + * note: queue'size is small, default 1 + */ +void transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)); + +/* + * put arg into queue + * if queue'size > 1, return false; else return true + */ +bool transQueuePush(STransQueue* queue, void* arg); +/* + * the size of queue + */ +int32_t transQueueSize(STransQueue* queue); +/* + * pop head from queue + */ +void* transQueuePop(STransQueue* queue); +/* + * get ith from queue + */ +void* transQueueGet(STransQueue* queue, int i); +/* + * rm ith from queue + */ +void* transQueueRm(STransQueue* queue, int i); +/* + * queue empty or not + */ +bool transQueueEmpty(STransQueue* queue); +/* + * clear queue + */ +void transQueueClear(STransQueue* queue); +/* + * destroy queue + */ +void transQueueDestroy(STransQueue* queue); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index e7393804674341417e6589d5b6f99961d39e856d..ad50948a021ac663bcc012ff6cbf08f86a354e4e 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -63,9 +63,6 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); - bool (*pfp)(void* parent, tmsg_t msgType); - void* (*mfp)(void* parent, tmsg_t msgType); - bool (*efp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; @@ -73,7 +70,7 @@ typedef struct { void* tmrCtrl; // handle to timer SHashObj* hash; // handle returned by hash utility void* tcphandle; // returned handle from TCP initialization - pthread_mutex_t mutex; + TdThreadMutex mutex; } SRpcInfo; #endif // USE_LIBUV diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 1db2808126a162e4b15305ff80057955de1025a3..e02de2d8b96f628d8eb07355690b968840378863 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -39,7 +39,7 @@ typedef struct { int total; int * count; int64_t keepTimer; - pthread_mutex_t mutex; + TdThreadMutex mutex; void (*cleanFp)(void *); void * tmrCtrl; void * pTimer; @@ -60,21 +60,21 @@ void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, connHashMemPool = taosMemPoolInit(maxSessions, sizeof(SConnHash)); if (connHashMemPool == 0) return NULL; - connHashList = calloc(sizeof(SConnHash *), maxSessions); + connHashList = taosMemoryCalloc(sizeof(SConnHash *), maxSessions); if (connHashList == 0) { taosMemPoolCleanUp(connHashMemPool); return NULL; } - pCache = malloc(sizeof(SConnCache)); + pCache = taosMemoryMalloc(sizeof(SConnCache)); if (pCache == NULL) { taosMemPoolCleanUp(connHashMemPool); - free(connHashList); + taosMemoryFree(connHashList); return NULL; } memset(pCache, 0, sizeof(SConnCache)); - pCache->count = calloc(sizeof(int), maxSessions); + pCache->count = taosMemoryCalloc(sizeof(int), maxSessions); pCache->total = 0; pCache->keepTimer = keepTimer; pCache->maxSessions = maxSessions; @@ -82,10 +82,10 @@ void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, pCache->connHashList = connHashList; pCache->cleanFp = cleanFp; pCache->tmrCtrl = tmrCtrl; - pCache->lockedBy = calloc(sizeof(int64_t), maxSessions); + pCache->lockedBy = taosMemoryCalloc(sizeof(int64_t), maxSessions); taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); - pthread_mutex_init(&pCache->mutex, NULL); + taosThreadMutexInit(&pCache->mutex, NULL); return pCache; } @@ -96,22 +96,22 @@ void rpcCloseConnCache(void *handle) { pCache = (SConnCache *)handle; if (pCache == NULL || pCache->maxSessions == 0) return; - pthread_mutex_lock(&pCache->mutex); + taosThreadMutexLock(&pCache->mutex); taosTmrStopA(&(pCache->pTimer)); if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); - tfree(pCache->connHashList); - tfree(pCache->count); - tfree(pCache->lockedBy); + taosMemoryFreeClear(pCache->connHashList); + taosMemoryFreeClear(pCache->count); + taosMemoryFreeClear(pCache->lockedBy); - pthread_mutex_unlock(&pCache->mutex); + taosThreadMutexUnlock(&pCache->mutex); - pthread_mutex_destroy(&pCache->mutex); + taosThreadMutexDestroy(&pCache->mutex); memset(pCache, 0, sizeof(SConnCache)); - free(pCache); + taosMemoryFree(pCache); } void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType) { @@ -220,7 +220,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { if (pCache == NULL || pCache->maxSessions == 0) return; if (pCache->pTimer != tmrId) return; - pthread_mutex_lock(&pCache->mutex); + taosThreadMutexLock(&pCache->mutex); uint64_t time = taosGetTimestampMs(); for (hash = 0; hash < pCache->maxSessions; ++hash) { @@ -232,7 +232,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { // tTrace("timer, total connections in cache:%d", pCache->total); taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); - pthread_mutex_unlock(&pCache->mutex); + taosThreadMutexUnlock(&pCache->mutex); } static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) { diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 86dd17bb0c8978c04d3d031f827b1b23d5a78d43..6beda5bfed11c3a42bf03457b000bf295792d673 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -14,6 +14,7 @@ */ #include "lz4.h" +#include "transportInt.h" #include "os.h" #include "rpcCache.h" #include "rpcHead.h" @@ -27,13 +28,12 @@ #include "tmd5.h" #include "tmempool.h" #include "tmsg.h" -#include "transportInt.h" #include "tref.h" #include "trpc.h" #include "ttimer.h" #include "tutil.h" -static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; +static TdThreadOnce tsRpcInitOnce = PTHREAD_ONCE_INIT; int tsRpcMaxUdpSize = 15000; // bytes int tsProgressTimer = 100; @@ -72,7 +72,7 @@ typedef struct { void * tcphandle; // returned handle from TCP initialization void * udphandle; // returned handle from UDP initialization void * pCache; // connection cache - pthread_mutex_t mutex; + TdThreadMutex mutex; struct SRpcConn *connList; // connection list } SRpcInfo; @@ -143,7 +143,7 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; -// static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; +// static TdThreadOnce tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 #define RPC_CONN_UDPS 0 @@ -208,7 +208,7 @@ static void rpcDecRef(SRpcInfo *pRpc); static void rpcFree(void *p) { tTrace("free mem: %p", p); - free(p); + taosMemoryFree(p); } static void rpcInitImp(void) { @@ -223,7 +223,7 @@ static void rpcInitImp(void) { } int32_t rpcInit() { - pthread_once(&tsRpcInitOnce, rpcInitImp); + taosThreadOnce(&tsRpcInitOnce, rpcInitImp); return 0; } @@ -238,9 +238,9 @@ void rpcCleanup(void) { void *rpcOpen(const SRpcInit *pInit) { SRpcInfo *pRpc; - // pthread_once(&tsRpcInit, rpcInit); + // taosThreadOnce(&tsRpcInit, rpcInit); - pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); + pRpc = (SRpcInfo *)taosMemoryCalloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; if (pInit->label) tstrncpy(pRpc->label, pInit->label, tListLen(pInit->label)); @@ -270,7 +270,7 @@ void *rpcOpen(const SRpcInit *pInit) { atomic_add_fetch_32(&tsRpcNum, 1); size_t size = sizeof(SRpcConn) * pRpc->sessions; - pRpc->connList = (SRpcConn *)calloc(1, size); + pRpc->connList = (SRpcConn *)taosMemoryCalloc(1, size); if (pRpc->connList == NULL) { tError("%s failed to allocate memory for taos connections, size:%" PRId64, pRpc->label, (int64_t)size); rpcClose(pRpc); @@ -307,7 +307,7 @@ void *rpcOpen(const SRpcInit *pInit) { } } - pthread_mutex_init(&pRpc->mutex, NULL); + taosThreadMutexInit(&pRpc->mutex, NULL); pRpc->tcphandle = (*taosInitConn[pRpc->connType | RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); @@ -350,7 +350,7 @@ void rpcClose(void *param) { void *rpcMallocCont(int contLen) { int size = contLen + RPC_MSG_OVERHEAD; - char *start = (char *)calloc(1, (size_t)size); + char *start = (char *)taosMemoryCalloc(1, (size_t)size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); return NULL; @@ -364,7 +364,7 @@ void *rpcMallocCont(int contLen) { void rpcFreeCont(void *cont) { if (cont) { char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext); - free(temp); + taosMemoryFree(temp); tTrace("free mem: %p", temp); } } @@ -374,12 +374,12 @@ void *rpcReallocCont(void *ptr, int contLen) { char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead); if (contLen == 0) { - free(start); + taosMemoryFree(start); return NULL; } int size = contLen + RPC_MSG_OVERHEAD; - start = realloc(start, size); + start = taosMemoryRealloc(start, size); if (start == NULL) { tError("failed to realloc cont, size:%d", size); return NULL; @@ -574,7 +574,7 @@ void rpcCancelRequest(int64_t rid) { static void rpcFreeMsg(void *msg) { if (msg) { char *temp = (char *)msg - sizeof(SRpcReqContext); - free(temp); + taosMemoryFree(temp); tTrace("free mem: %p", temp); } } @@ -1039,7 +1039,7 @@ static void doRpcReportBrokenLinkToServer(void *param, void *id) { SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); SRpcInfo *pRpc = pConn->pRpc; (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL); - free(pRpcMsg); + taosMemoryFree(pRpcMsg); } static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; @@ -1049,7 +1049,7 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { rpcAddRef(pRpc); tDebug("%s, notify the server app, connection is gone", pConn->info); - SRpcMsg *rpcMsg = malloc(sizeof(SRpcMsg)); + SRpcMsg *rpcMsg = taosMemoryMalloc(sizeof(SRpcMsg)); rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length rpcMsg->ahandle = pConn->ahandle; @@ -1061,7 +1061,7 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { if (pRpc->cfp) { taosTmrStart(doRpcReportBrokenLinkToServer, 0, rpcMsg, pRpc->tmrCtrl); } else { - free(rpcMsg); + taosMemoryFree(rpcMsg); } } @@ -1484,7 +1484,7 @@ static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { return contLen; } - char *buf = malloc(contLen + overhead + 8); // 8 extra bytes + char *buf = taosMemoryMalloc(contLen + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); return contLen; @@ -1510,7 +1510,7 @@ static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { finalLen = contLen; } - free(buf); + taosMemoryFree(buf); return finalLen; } @@ -1526,7 +1526,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { int contLen = htonl(pComp->contLen); // prepare the temporary buffer to decompress message - char *temp = (char *)malloc(contLen + RPC_MSG_OVERHEAD); + char *temp = (char *)taosMemoryMalloc(contLen + RPC_MSG_OVERHEAD); pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext if (pNewHead) { @@ -1671,10 +1671,10 @@ static void rpcDecRef(SRpcInfo *pRpc) { taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); - tfree(pRpc->connList); - pthread_mutex_destroy(&pRpc->mutex); + taosMemoryFreeClear(pRpc->connList); + taosThreadMutexDestroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); - tfree(pRpc); + taosMemoryFreeClear(pRpc); atomic_sub_fetch_32(&tsRpcNum, 1); } diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index aac38b21e8fdf570dfe76414586e6cca83961dd7..52c5ddcf631e9a0cc3a307fab00122c5139fa5dd 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -35,9 +35,9 @@ typedef struct SFdObj { } SFdObj; typedef struct SThreadObj { - pthread_t thread; + TdThread thread; SFdObj * pHead; - pthread_mutex_t mutex; + TdThreadMutex mutex; uint32_t ip; bool stop; TdEpollPtr pEpoll; @@ -65,7 +65,7 @@ typedef struct { int numOfThreads; void * shandle; SThreadObj **pThreadObj; - pthread_t thread; + TdThread thread; } SServerObj; static void * taosProcessTcpData(void *param); @@ -78,7 +78,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread SServerObj *pServerObj; SThreadObj *pThreadObj; - pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); + pServerObj = (SServerObj *)taosMemoryCalloc(sizeof(SServerObj), 1); if (pServerObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -92,28 +92,28 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pServerObj->label, label, sizeof(pServerObj->label)); pServerObj->numOfThreads = numOfThreads; - pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads); + pServerObj->pThreadObj = (SThreadObj **)taosMemoryCalloc(sizeof(SThreadObj *), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - free(pServerObj); + taosMemoryFree(pServerObj); return NULL; } int code = 0; - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); // initialize parameters in case it may encounter error later for (int i = 0; i < numOfThreads; ++i) { - pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1); + pThreadObj = (SThreadObj *)taosMemoryCalloc(sizeof(SThreadObj), 1); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j = 0; j < i; ++j) free(pServerObj->pThreadObj[j]); - free(pServerObj->pThreadObj); - free(pServerObj); + for (int j = 0; j < i; ++j) taosMemoryFree(pServerObj->pThreadObj[j]); + taosMemoryFree(pServerObj->pThreadObj); + taosMemoryFree(pServerObj); return NULL; } @@ -129,7 +129,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread // initialize mutex, thread, fd which may fail for (int i = 0; i < numOfThreads; ++i) { pThreadObj = pServerObj->pThreadObj[i]; - code = pthread_mutex_init(&(pThreadObj->mutex), NULL); + code = taosThreadMutexInit(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break; @@ -142,7 +142,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + code = taosThreadCreate(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); break; @@ -155,7 +155,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread if (pServerObj->pSocketServer == NULL) code = -1; if (code == 0) { - code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); + code = taosThreadCreate(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); if (code != 0) { tError("%s failed to create TCP accept thread(%s)", label, strerror(code)); } @@ -169,7 +169,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread tDebug("%s TCP server is initialized, ip:0x%x port:%hu numOfThreads:%d", label, ip, port, numOfThreads); } - pthread_attr_destroy(&thattr); + taosThreadAttrDestroy(&thattr); return (void *)pServerObj; } @@ -178,16 +178,16 @@ static void taosStopTcpThread(SThreadObj *pThreadObj) { return; } // save thread into local variable and signal thread to stop - pthread_t thread = pThreadObj->thread; + TdThread thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { return; } pThreadObj->stop = true; - if (taosComparePthread(thread, pthread_self())) { - pthread_detach(pthread_self()); + if (taosComparePthread(thread, taosThreadSelf())) { + pthread_detach(taosThreadSelf()); return; } - pthread_join(thread, NULL); + taosThreadJoin(thread, NULL); } void taosStopTcpServer(void *handle) { @@ -200,10 +200,10 @@ void taosStopTcpServer(void *handle) { taosShutDownSocketServerRD(pServerObj->pSocketServer); } if (taosCheckPthreadValid(pServerObj->thread)) { - if (taosComparePthread(pServerObj->thread, pthread_self())) { - pthread_detach(pthread_self()); + if (taosComparePthread(pServerObj->thread, taosThreadSelf())) { + pthread_detach(taosThreadSelf()); } else { - pthread_join(pServerObj->thread, NULL); + taosThreadJoin(pServerObj->thread, NULL); } } @@ -222,8 +222,8 @@ void taosCleanUpTcpServer(void *handle) { tDebug("%s TCP server is cleaned up", pServerObj->label); - tfree(pServerObj->pThreadObj); - tfree(pServerObj); + taosMemoryFreeClear(pServerObj->pThreadObj); + taosMemoryFreeClear(pServerObj); } static void *taosAcceptTcpConnection(void *arg) { @@ -290,7 +290,7 @@ static void *taosAcceptTcpConnection(void *arg) { } void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - SClientObj *pClientObj = (SClientObj *)calloc(1, sizeof(SClientObj)); + SClientObj *pClientObj = (SClientObj *)taosMemoryCalloc(1, sizeof(SClientObj)); if (pClientObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -299,26 +299,26 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); pClientObj->numOfThreads = numOfThreads; - pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj *)); + pClientObj->pThreadObj = (SThreadObj **)taosMemoryCalloc(numOfThreads, sizeof(SThreadObj *)); if (pClientObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - tfree(pClientObj); + taosMemoryFreeClear(pClientObj); terrno = TAOS_SYSTEM_ERROR(errno); } int code = 0; - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < numOfThreads; ++i) { - SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj)); + SThreadObj *pThreadObj = (SThreadObj *)taosMemoryCalloc(1, sizeof(SThreadObj)); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]); - free(pClientObj); - pthread_attr_destroy(&thattr); + for (int j = 0; j < i; ++j) taosMemoryFree(pClientObj->pThreadObj[j]); + taosMemoryFree(pClientObj); + taosThreadAttrDestroy(&thattr); return NULL; } pClientObj->pThreadObj[i] = pThreadObj; @@ -333,7 +333,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread // initialize mutex, thread, fd which may fail for (int i = 0; i < numOfThreads; ++i) { SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; - code = pthread_mutex_init(&(pThreadObj->mutex), NULL); + code = taosThreadMutexInit(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break; @@ -346,7 +346,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + code = taosThreadCreate(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); break; @@ -378,8 +378,8 @@ void taosCleanUpTcpClient(void *chandle) { } tDebug("%s TCP client is cleaned up", pClientObj->label); - tfree(pClientObj->pThreadObj); - tfree(pClientObj); + taosMemoryFreeClear(pClientObj->pThreadObj); + taosMemoryFreeClear(pClientObj); } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { @@ -477,9 +477,9 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); int32_t size = msgLen + tsRpcOverhead; - buffer = malloc(size); + buffer = taosMemoryMalloc(size); if (NULL == buffer) { - tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); + tError("%s %p TCP taosMemoryMalloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { tTrace("%s %p read data, FD:%p TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, buffer); @@ -491,7 +491,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { if (leftLen != retLen) { tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); - free(buffer); + taosMemoryFree(buffer); return -1; } @@ -507,7 +507,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { pInfo->connType = RPC_CONN_TCP; if (pFdObj->closedByApp) { - free(buffer); + taosMemoryFree(buffer); return -1; } @@ -578,9 +578,9 @@ static void *taosProcessTcpData(void *param) { taosReportBrokenLink(pFdObj); } - pthread_mutex_destroy(&(pThreadObj->mutex)); + taosThreadMutexDestroy(&(pThreadObj->mutex)); tDebug("%s TCP thread exits ...", pThreadObj->label); - tfree(pThreadObj); + taosMemoryFreeClear(pThreadObj); return NULL; } @@ -588,7 +588,7 @@ static void *taosProcessTcpData(void *param) { static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { struct epoll_event event; - SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); + SFdObj *pFdObj = (SFdObj *)taosMemoryCalloc(sizeof(SFdObj), 1); if (pFdObj == NULL) { return NULL; } @@ -601,18 +601,18 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; if (taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_ADD, pSocket, &event) < 0) { - tfree(pFdObj); + taosMemoryFreeClear(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->mutex)); + taosThreadMutexLock(&(pThreadObj->mutex)); pFdObj->next = pThreadObj->pHead; if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; pThreadObj->pHead = pFdObj; pThreadObj->numOfFds++; - pthread_mutex_unlock(&(pThreadObj->mutex)); + taosThreadMutexUnlock(&(pThreadObj->mutex)); return pFdObj; } @@ -622,10 +622,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) { if (pFdObj->signature != pFdObj) return; SThreadObj *pThreadObj = pFdObj->pThreadObj; - pthread_mutex_lock(&pThreadObj->mutex); + taosThreadMutexLock(&pThreadObj->mutex); if (pFdObj->signature == NULL) { - pthread_mutex_unlock(&pThreadObj->mutex); + taosThreadMutexUnlock(&pThreadObj->mutex); return; } @@ -648,10 +648,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) { (pFdObj->next)->prev = pFdObj->prev; } - pthread_mutex_unlock(&pThreadObj->mutex); + taosThreadMutexUnlock(&pThreadObj->mutex); tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); - tfree(pFdObj); + taosMemoryFreeClear(pFdObj); } #endif \ No newline at end of file diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 81c8d0af7616598c886ce9dad9fb5be16c83c117..359a94011dc493493aa9c63660806787b3be4d11 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -35,7 +35,7 @@ typedef struct { uint16_t port; // peer port uint16_t localPort; // local port char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - pthread_t thread; + TdThread thread; void *hash; void *shandle; // handle passed by upper layer during server initialization void *pSet; @@ -62,7 +62,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); - pSet = (SUdpConnSet *)malloc((size_t)size); + pSet = (SUdpConnSet *)taosMemoryMalloc((size_t)size); if (pSet == NULL) { tError("%s failed to allocate UdpConn", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -77,9 +77,9 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pSet->threads = threads; tstrncpy(pSet->label, label, sizeof(pSet->label)); - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); int i; uint16_t ownPort; @@ -92,7 +92,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads break; } - pConn->buffer = malloc(RPC_MAX_UDP_SIZE); + pConn->buffer = taosMemoryMalloc(RPC_MAX_UDP_SIZE); if (NULL == pConn->buffer) { tError("%s failed to malloc recv buffer", label); break; @@ -111,14 +111,14 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pConn->index = i; pConn->pSet = pSet; - int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); + int code = taosThreadCreate(&pConn->thread, &thAttr, taosRecvUdpData, pConn); if (code != 0) { tError("%s failed to create thread to process UDP data(%s)", label, strerror(errno)); break; } } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); if (i != threads) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -146,9 +146,9 @@ void taosStopUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; if (taosCheckPthreadValid(pConn->thread)) { - pthread_join(pConn->thread, NULL); + taosThreadJoin(pConn->thread, NULL); } - tfree(pConn->buffer); + taosMemoryFreeClear(pConn->buffer); // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); } @@ -167,7 +167,7 @@ void taosCleanUpUdpConnection(void *handle) { } tDebug("%s UDP is cleaned up", pSet->label); - tfree(pSet); + taosMemoryFreeClear(pSet); } void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { @@ -219,7 +219,7 @@ static void *taosRecvUdpData(void *param) { } int32_t size = dataLen + tsRpcOverhead; - char * tmsg = malloc(size); + char * tmsg = taosMemoryMalloc(size); if (NULL == tmsg) { tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6d1c691b9ba17ac001bfcdf35212cc225f11b0d6..90f15dd7d052b637f473e56a0dfb88c9963a987b 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -14,10 +14,13 @@ */ #define _DEFAULT_SOURCE +#ifdef USE_UV +#include +#endif +#include "zlib.h" #include "thttp.h" #include "taoserror.h" #include "tlog.h" -#include "zlib.h" static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pHead, int32_t headLen, EHttpCompFlag flag) { @@ -44,7 +47,7 @@ static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pH int32_t taosCompressHttpRport(char* pSrc, int32_t srcLen) { int32_t code = -1; int32_t destLen = srcLen; - void* pDest = malloc(destLen); + void* pDest = taosMemoryMalloc(destLen); if (pDest == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -106,12 +109,11 @@ _OVER: code = gzipStream.total_out; } - free(pDest); + taosMemoryFree(pDest); return code; } #ifdef USE_UV -#include static void clientConnCb(uv_connect_t* req, int32_t status) { if (status < 0) { terrno = TAOS_SYSTEM_ERROR(status); @@ -143,7 +145,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 uv_tcp_t socket_tcp = {0}; uv_loop_t* loop = uv_default_loop(); uv_tcp_init(loop, &socket_tcp); - uv_connect_t* connect = (uv_connect_t*)malloc(sizeof(uv_connect_t)); + uv_connect_t* connect = (uv_connect_t*)taosMemoryMalloc(sizeof(uv_connect_t)); if (flag == HTTP_GZIP) { int32_t dstLen = taosCompressHttpRport(pCont, contLen); @@ -166,7 +168,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 terrno = 0; uv_run(loop, UV_RUN_DEFAULT); uv_loop_close(loop); - free(connect); + taosMemoryFree(connect); return terrno; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 2cab03f133620fce7d2129f251320946bcc010ea..ebb90338cd42303243f44be81bceef25f1160b63 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -28,7 +28,7 @@ void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHan void (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; void* rpcOpen(const SRpcInit* pInit) { - SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) { return NULL; } @@ -39,9 +39,6 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; - pRpc->pfp = pInit->pfp; - pRpc->mfp = pInit->mfp; - pRpc->efp = pInit->efp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; @@ -64,13 +61,13 @@ void* rpcOpen(const SRpcInit* pInit) { void rpcClose(void* arg) { SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); - free(pRpc); + taosMemoryFree(pRpc); return; } void* rpcMallocCont(int contLen) { int size = contLen + TRANS_MSG_OVERHEAD; - char* start = (char*)calloc(1, (size_t)size); + char* start = (char*)taosMemoryCalloc(1, (size_t)size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); return NULL; @@ -84,7 +81,7 @@ void rpcFreeCont(void* cont) { if (cont == NULL) { return; } - free((char*)cont - TRANS_MSG_OVERHEAD); + taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD); } void* rpcReallocCont(void* ptr, int contLen) { if (ptr == NULL) { @@ -92,7 +89,7 @@ void* rpcReallocCont(void* ptr, int contLen) { } char* st = (char*)ptr - TRANS_MSG_OVERHEAD; int sz = contLen + TRANS_MSG_OVERHEAD; - st = realloc(st, sz); + st = taosMemoryRealloc(st, sz); if (st == NULL) { return NULL; } @@ -121,7 +118,12 @@ void rpcCancelRequest(int64_t rid) { return; } void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); uint32_t port = pEpSet->eps[pEpSet->inUse].port; - transSendRequest(shandle, ip, port, pMsg); + transSendRequest(shandle, ip, port, pMsg, NULL); +} +void rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { + char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); + uint32_t port = pEpSet->eps[pEpSet->inUse].port; + transSendRequest(shandle, ip, port, pMsg, pCtx); } void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); @@ -142,6 +144,7 @@ void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } +void rpcRegisterBrokenLinkArg(SRpcMsg* msg) { transRegisterMsg(msg); } void rpcReleaseHandle(void* handle, int8_t type) { assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); (*transReleaseHandle[type])(handle); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index df35b8d67b2e2b0c91c154c4ff3d7cb951d0d8b1..d9c288a39b92e2b673d3e834fcca3c7afaae8d47 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -17,11 +17,6 @@ #include "transComm.h" -// Normal(default): send/recv msg -// Quit: quit rpc inst -// Release: release handle to rpc inst -typedef enum { Normal, Quit, Release } SCliMsgType; - typedef struct SCliConn { T_REF_DECLARE() uv_connect_t connReq; @@ -30,13 +25,15 @@ typedef struct SCliConn { void* hostThrd; SConnBuffer readBuf; void* data; - SArray* cliMsgs; + STransQueue cliMsgs; queue conn; uint64_t expireTime; int hThrdIdx; - bool broken; // link broken or not + STransCtx ctx; - int persist; // + bool broken; // link broken or not + ConnStatus status; // + int release; // 1: release // spi configure char spi; char secured; @@ -55,19 +52,20 @@ typedef struct SCliMsg { STransMsg msg; queue q; uint64_t st; - SCliMsgType type; + STransMsgType type; + int sent; //(0: no send, 1: alread sent) } SCliMsg; typedef struct SCliThrdObj { - pthread_t thread; + TdThread thread; uv_loop_t* loop; SAsyncPool* asyncPool; uv_timer_t timer; void* pool; // conn pool // msg queue - queue msg; - pthread_mutex_t msgMtx; + queue msg; + TdThreadMutex msgMtx; uint64_t nextTimeout; // next timeout void* pTransInst; // @@ -113,10 +111,12 @@ static void cliSend(SCliConn* pConn); static void cliHandleResp(SCliConn* conn); // handle except about conn static void cliHandleExcept(SCliConn* conn); + // handle req from app static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease}; static void cliSendQuit(SCliThrdObj* thrd); static void destroyUserdata(STransMsg* userdata); @@ -133,42 +133,97 @@ static void destroyThrdObj(SCliThrdObj* pThrd); #define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) -#define CONN_HANDLE_THREAD_QUIT(conn, thrd) \ - do { \ - if (thrd->quit) { \ - cliHandleExcept(conn); \ - goto _RETURE; \ - } \ +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + uint64_t ahandle = head->ahandle; \ + CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tDebug("cli conn %p receive release request, ref: %d", conn, T_REF_VAL_GET(conn)); \ + while (T_REF_VAL_GET(conn) > 1) { \ + transUnrefCliHandle(conn); \ + } \ + if (T_REF_VAL_GET(conn) == 1) { \ + SCliThrdObj* thrd = conn->hostThrd; \ + addConnToPool(thrd->pool, conn); \ + } \ + destroyCmsg(pMsg); \ + return; \ + } \ + } while (0) + +#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \ + do { \ + int i = 0, sz = transQueueSize(&conn->cliMsgs); \ + for (; i < sz; i++) { \ + pMsg = transQueueGet(&conn->cliMsgs, i); \ + if (pMsg != NULL && pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \ + break; \ + } \ + } \ + if (i == sz) { \ + pMsg = NULL; \ + } else { \ + pMsg = transQueueRm(&conn->cliMsgs, i); \ + } \ + } while (0) +#define CONN_GET_NEXT_SENDMSG(conn) \ + do { \ + int i = 0; \ + do { \ + pCliMsg = transQueueGet(&conn->cliMsgs, i++); \ + if (pCliMsg && 0 == pCliMsg->sent) { \ + break; \ + } \ + } while (pCliMsg != NULL); \ + if (pCliMsg == NULL) { \ + goto _RETURN; \ + } \ + } while (0) + +#define CONN_HANDLE_THREAD_QUIT(thrd) \ + do { \ + if (thrd->quit) { \ + return; \ + } \ } while (0) #define CONN_HANDLE_BROKEN(conn) \ do { \ if (conn->broken) { \ cliHandleExcept(conn); \ - goto _RETURE; \ + return; \ } \ - } while (0); + } while (0) #define CONN_SET_PERSIST_BY_APP(conn) \ do { \ - if (conn->persist == false) { \ - conn->persist = true; \ + if (conn->status == ConnNormal) { \ + conn->status = ConnAcquire; \ transRefCliHandle(conn); \ } \ } while (0) -#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false) - +#define CONN_NO_PERSIST_BY_APP(conn) \ + (((conn)->status == ConnNormal || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) +#define CONN_RELEASE_BY_SERVER(conn) \ + (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) #define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) +#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) +#define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) static void* cliWorkThread(void* arg); bool cliMaySendCachedMsg(SCliConn* conn) { - if (taosArrayGetSize(conn->cliMsgs) > 0) { + if (!transQueueEmpty(&conn->cliMsgs)) { + SCliMsg* pCliMsg = NULL; + CONN_GET_NEXT_SENDMSG(conn); cliSend(conn); - return true; - } else { - return false; } + return false; +_RETURN: + return false; } void cliHandleResp(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; @@ -185,24 +240,44 @@ void cliHandleResp(SCliConn* conn) { transMsg.msgType = pHead->msgType; transMsg.ahandle = NULL; - SCliMsg* pMsg = NULL; - if (taosArrayGetSize(conn->cliMsgs) > 0) { - pMsg = taosArrayGetP(conn->cliMsgs, 0); - taosArrayRemove(conn->cliMsgs, 0); - } + SCliMsg* pMsg = NULL; + STransConnCtx* pCtx = NULL; + CONN_SHOULD_RELEASE(conn, pHead); - STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; - if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; + if (CONN_NO_PERSIST_BY_APP(conn)) { + pMsg = transQueuePop(&conn->cliMsgs); + pCtx = pMsg ? pMsg->ctx : NULL; + if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { + transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); + if (transMsg.ahandle == NULL) { + transMsg.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType)); + } + tDebug("cli conn %p construct ahandle %p, persist: 0", conn, transMsg.ahandle); + } else { + transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + tDebug("cli conn %p get ahandle %p, persist: 0", conn, transMsg.ahandle); + } } else { - transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + uint64_t ahandle = (uint64_t)pHead->ahandle; + CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); + if (pMsg == NULL) { + transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); + tDebug("cli conn %p construct ahandle %p by %d, persist: 1", conn, transMsg.ahandle, transMsg.msgType); + if (!CONN_RELEASE_BY_SERVER(conn) && transMsg.ahandle == NULL) { + transMsg.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType)); + tDebug("cli conn %p construct ahandle %p due brokenlink, persist: 1", conn, transMsg.ahandle); + } + } else { + pCtx = pMsg ? pMsg->ctx : NULL; + transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + tDebug("cli conn %p get ahandle %p, persist: 1", conn, transMsg.ahandle); + } } // buf's mem alread translated to transMsg.pCont transClearBuffer(&conn->readBuf); - if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, transMsg.msgType)) { + if (!CONN_NO_PERSIST_BY_APP(conn)) { transMsg.handle = conn; - CONN_SET_PERSIST_BY_APP(conn); tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } @@ -217,6 +292,11 @@ void cliHandleResp(SCliConn* conn) { // transUnrefCliHandle(conn); return; } + if (CONN_RELEASE_BY_SERVER(conn) && transMsg.ahandle == NULL) { + tTrace("except, server continue send while cli ignore it"); + // transUnrefCliHandle(conn); + return; + } if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, conn); @@ -244,46 +324,57 @@ void cliHandleResp(SCliConn* conn) { } void cliHandleExcept(SCliConn* pConn) { - if (taosArrayGetSize(pConn->cliMsgs) == 0) { - if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) { + if (transQueueEmpty(&pConn->cliMsgs)) { + if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { + tTrace("%s cli conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); transUnrefCliHandle(pConn); return; } } SCliThrdObj* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; - + bool once = false; do { - SCliMsg* pMsg = NULL; - if (taosArrayGetSize(pConn->cliMsgs) > 0) { - pMsg = taosArrayGetP(pConn->cliMsgs, 0); - taosArrayRemove(pConn->cliMsgs, 0); + SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs); + if (pMsg == NULL && once) { + break; } - STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; STransMsg transMsg = {0}; transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.ahandle = NULL; + transMsg.handle = pConn; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { - transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; + transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); + tDebug("%s cli conn %p construct ahandle %p by %s", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle, + TMSG_INFO(transMsg.msgType)); + if (transMsg.ahandle == NULL) { + transMsg.ahandle = transCtxDumpBrokenlinkVal(&pConn->ctx, (int32_t*)&(transMsg.msgType)); + tDebug("%s cli conn %p construct ahandle %p due to brokenlink", CONN_GET_INST_LABEL(pConn), pConn, + transMsg.ahandle); + } } else { transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } if (pCtx == NULL || pCtx->pSem == NULL) { - tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); + tTrace("%s cli conn %p handle except", pTransInst->label, pConn); + if (transMsg.ahandle == NULL) { + once = true; + continue; + } (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { - tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, pConn); + tTrace("%s cli conn(sync) %p handle except", pTransInst->label, pConn); memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); tsem_post(pCtx->pSem); } destroyCmsg(pMsg); tTrace("%s cli conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); - } while (taosArrayGetSize(pConn->cliMsgs) > 0); + } while (!transQueueEmpty(&pConn->cliMsgs)); transUnrefCliHandle(pConn); } @@ -351,31 +442,35 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { } queue* h = QUEUE_HEAD(&plist->conn); QUEUE_REMOVE(h); - SCliConn* conn = QUEUE_DATA(h, SCliConn, conn); + conn->status = ConnNormal; QUEUE_INIT(&conn->conn); return conn; } static void addConnToPool(void* pool, SCliConn* conn) { - char key[128] = {0}; + SCliThrdObj* thrd = conn->hostThrd; + CONN_HANDLE_THREAD_QUIT(thrd); + STrans* pTransInst = ((SCliThrdObj*)conn->hostThrd)->pTransInst; + conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); + transCtxCleanup(&conn->ctx); + transQueueClear(&conn->cliMsgs); + conn->status = ConnInPool; + + char key[128] = {0}; tstrncpy(key, conn->ip, strlen(conn->ip)); tstrncpy(key + strlen(key), (char*)(&conn->port), sizeof(conn->port)); tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); - STrans* pTransInst = ((SCliThrdObj*)conn->hostThrd)->pTransInst; - - conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); + assert(!QUEUE_IS_EMPTY(&plist->conn)); } static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; - // avoid conn - QUEUE_REMOVE(&conn->conn); transAllocBuffer(pBuf, buf); } static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { @@ -401,6 +496,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb // nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under // read(2). + tTrace("%s cli conn %p read empty", CONN_GET_INST_LABEL(conn), conn); return; } if (nread < 0) { @@ -411,25 +507,26 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { } static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { - SCliConn* conn = calloc(1, sizeof(SCliConn)); + SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn)); // read/write stream handle - conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + conn->stream = (uv_stream_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->stream->data = conn; conn->writeReq.data = conn; conn->connReq.data = conn; - conn->cliMsgs = taosArrayInit(2, sizeof(void*)); + transQueueInit(&conn->cliMsgs, NULL); QUEUE_INIT(&conn->conn); conn->hostThrd = pThrd; - conn->persist = false; - conn->broken = false; + conn->status = ConnNormal; + conn->broken = 0; transRefCliHandle(conn); return conn; } static void cliDestroyConn(SCliConn* conn, bool clear) { tTrace("%s cli conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); + QUEUE_REMOVE(&conn->conn); if (clear) { uv_close((uv_handle_t*)conn->stream, cliDestroy); @@ -437,19 +534,20 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } static void cliDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; - free(conn->ip); - free(conn->stream); - taosArrayDestroy(conn->cliMsgs); + taosMemoryFree(conn->ip); + taosMemoryFree(conn->stream); + transCtxCleanup(&conn->ctx); + transQueueDestroy(&conn->cliMsgs); tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); - free(conn); + taosMemoryFree(conn); } static bool cliHandleNoResp(SCliConn* conn) { - bool res = false; - SArray* msgs = conn->cliMsgs; - if (taosArrayGetSize(msgs) > 0) { - SCliMsg* pMsg = taosArrayGetP(msgs, 0); + bool res = false; + if (!transQueueEmpty(&conn->cliMsgs)) { + SCliMsg* pMsg = transQueueGet(&conn->cliMsgs, 0); if (REQUEST_NO_RESP(&pMsg->msg)) { - taosArrayRemove(msgs, 0); + transQueuePop(&conn->cliMsgs); + // taosArrayRemove(msgs, 0); destroyCmsg(pMsg); res = true; } @@ -482,20 +580,30 @@ static void cliSendCb(uv_write_t* req, int status) { void cliSend(SCliConn* pConn) { CONN_HANDLE_BROKEN(pConn); - assert(taosArrayGetSize(pConn->cliMsgs) > 0); - SCliMsg* pCliMsg = taosArrayGetP(pConn->cliMsgs, 0); + // assert(taosArrayGetSize(pConn->cliMsgs) > 0); + assert(!transQueueEmpty(&pConn->cliMsgs)); + + SCliMsg* pCliMsg = NULL; + CONN_GET_NEXT_SENDMSG(pConn); + pCliMsg->sent = 1; + STransConnCtx* pCtx = pCliMsg->ctx; SCliThrdObj* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); - + if (pMsg->pCont == 0) { + pMsg->pCont = (void*)rpcMallocCont(0); + pMsg->contLen = 0; + } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - int msgLen = transMsgLenFromCont(pMsg->contLen); + pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; + + int msgLen = transMsgLenFromCont(pMsg->contLen); if (!pConn->secured) { - char* buf = calloc(1, msgLen + sizeof(STransUserMsg)); + char* buf = taosMemoryCalloc(1, msgLen + sizeof(STransUserMsg)); memcpy(buf, (char*)pHead, msgLen); STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen); @@ -513,20 +621,26 @@ void cliSend(SCliConn* pConn) { msgLen += sizeof(STransUserMsg); } - pHead->resflag = REQUEST_NO_RESP(pMsg) ? 1 : 0; + pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; + pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; pHead->msgType = pMsg->msgType; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + if (pHead->persist == 1) { + CONN_SET_PERSIST_BY_APP(pConn); + } + pConn->writeReq.data = pConn; uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); return; -_RETURE: +_RETURN: return; } @@ -562,20 +676,16 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { } static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = pMsg->msg.handle; - tDebug("%s cli conn %p release to inst", CONN_GET_INST_LABEL(conn), conn); - - while (taosArrayGetSize(conn->cliMsgs) > 0) { - SCliMsg* pMsg = taosArrayGetP(conn->cliMsgs, 0); - destroyCmsg(pMsg); - taosArrayRemove(conn->cliMsgs, 0); - } + tDebug("%s cli conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); - transDestroyBuffer(&conn->readBuf); - if (conn->persist && T_REF_VAL_GET(conn) >= 2) { - conn->persist = false; + if (T_REF_VAL_GET(conn) == 2) { transUnrefCliHandle(conn); - addConnToPool(pThrd->pool, conn); + if (!transQueuePush(&conn->cliMsgs, pMsg)) { + return; + } + cliSend(conn); } else { + // conn already broken down transUnrefCliHandle(conn); } } @@ -592,6 +702,8 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { tTrace("%s cli conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + } else { + tTrace("not found conn in conn pool %p", pThrd->pool); } } return conn; @@ -609,16 +721,16 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { if (conn != NULL) { conn->hThrdIdx = pCtx->hThrdIdx; - if (taosArrayGetSize(conn->cliMsgs) > 0) { - taosArrayPush(conn->cliMsgs, &pMsg); + transCtxMerge(&conn->ctx, &pCtx->appCtx); + if (!transQueuePush(&conn->cliMsgs, pMsg)) { return; } - taosArrayPush(conn->cliMsgs, &pMsg); transDestroyBuffer(&conn->readBuf); cliSend(conn); } else { conn = cliCreateConn(pThrd); - taosArrayPush(conn->cliMsgs, &pMsg); + transCtxMerge(&conn->ctx, &pCtx->appCtx); + transQueuePush(&conn->cliMsgs, pMsg); conn->hThrdIdx = pCtx->hThrdIdx; conn->ip = strdup(pMsg->ctx->ip); @@ -642,9 +754,9 @@ static void cliAsyncCb(uv_async_t* handle) { // batch process to avoid to lock/unlock frequently queue wq; - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_MOVE(&item->qmsg, &wq); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); int count = 0; while (!QUEUE_IS_EMPTY(&wq)) { @@ -652,14 +764,10 @@ static void cliAsyncCb(uv_async_t* handle) { QUEUE_REMOVE(h); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - - if (pMsg->type == Normal) { - cliHandleReq(pMsg, pThrd); - } else if (pMsg->type == Quit) { - cliHandleQuit(pMsg, pThrd); - } else if (pMsg->type == Release) { - cliHandleRelease(pMsg, pThrd); + if (pMsg == NULL) { + continue; } + (*cliAsyncHandle[pMsg->type])(pMsg, pThrd); count++; } if (count >= 2) { @@ -676,19 +784,19 @@ static void* cliWorkThread(void* arg) { } void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SCliObj* cli = calloc(1, sizeof(SCliObj)); + SCliObj* cli = taosMemoryCalloc(1, sizeof(SCliObj)); STrans* pTransInst = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; - cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); + cli->pThreadObj = (SCliThrdObj**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrdObj*)); for (int i = 0; i < cli->numOfThreads; i++) { SCliThrdObj* pThrd = createThrdObj(); pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = shandle; - int err = pthread_create(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); + int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); if (err == 0) { tDebug("success to create tranport-cli thread %d", i); } @@ -710,16 +818,16 @@ static void destroyCmsg(SCliMsg* pMsg) { } transDestroyConnCtx(pMsg->ctx); destroyUserdata(&pMsg->msg); - free(pMsg); + taosMemoryFree(pMsg); } static SCliThrdObj* createThrdObj() { - SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + SCliThrdObj* pThrd = (SCliThrdObj*)taosMemoryCalloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); + taosThreadMutexInit(&pThrd->msgMtx, NULL); - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, cliAsyncCb); @@ -737,25 +845,25 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { return; } uv_stop(pThrd->loop); - pthread_join(pThrd->thread, NULL); - pthread_mutex_destroy(&pThrd->msgMtx); + taosThreadJoin(pThrd->thread, NULL); + taosThreadMutexDestroy(&pThrd->msgMtx); transDestroyAsyncPool(pThrd->asyncPool); uv_timer_stop(&pThrd->timer); - free(pThrd->loop); - free(pThrd); + taosMemoryFree(pThrd->loop); + taosMemoryFree(pThrd); } static void transDestroyConnCtx(STransConnCtx* ctx) { if (ctx != NULL) { - free(ctx->ip); + taosMemoryFree(ctx->ip); } - free(ctx); + taosMemoryFree(ctx); } // void cliSendQuit(SCliThrdObj* thrd) { // cli can stop gracefully - SCliMsg* msg = calloc(1, sizeof(SCliMsg)); + SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; transSendAsync(thrd->asyncPool, &msg->q); } @@ -774,8 +882,8 @@ void transCloseClient(void* arg) { cliSendQuit(cli->pThreadObj[i]); destroyThrdObj(cli->pThreadObj[i]); } - free(cli->pThreadObj); - free(cli); + taosMemoryFree(cli->pThreadObj); + taosMemoryFree(cli); } void transRefCliHandle(void* handle) { if (handle == NULL) { @@ -801,42 +909,44 @@ void transReleaseCliHandle(void* handle) { } STransMsg tmsg = {.handle = handle}; - SCliMsg* cmsg = calloc(1, sizeof(SCliMsg)); - cmsg->type = Release; + SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; + cmsg->type = Release; transSendAsync(thrd->asyncPool, &cmsg->q); } -void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg) { +void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* ctx) { STrans* pTransInst = (STrans*)shandle; int index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle); if (index == -1) { index = cliRBChoseIdx(pTransInst); } - int32_t flen = 0; - if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) { - // imp later - } - tDebug("send request at thread:%d %p", index, pMsg); - STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + + STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; pCtx->ip = strdup(ip); pCtx->port = port; pCtx->hThrdIdx = index; + if (ctx != NULL) { + pCtx->appCtx = *ctx; + } assert(pTransInst->connType == TAOS_CONN_CLIENT); - // atomic or not - SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); + SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pMsg; cliMsg->st = taosGetTimestampUs(); + cliMsg->type = Normal; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; + + tDebug("send request at thread:%d %p, dst: %s:%d", index, pMsg, ip, port); transSendAsync(thrd->asyncPool, &(cliMsg->q)); } + void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)shandle; int index = CONN_HOST_THREAD_INDEX(pReq->handle); @@ -844,27 +954,28 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq index = cliRBChoseIdx(pTransInst); } - STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pReq->ahandle; pCtx->msgType = pReq->msgType; pCtx->ip = strdup(ip); pCtx->port = port; pCtx->hThrdIdx = index; - pCtx->pSem = calloc(1, sizeof(tsem_t)); + pCtx->pSem = taosMemoryCalloc(1, sizeof(tsem_t)); pCtx->pRsp = pRsp; tsem_init(pCtx->pSem, 0, 0); - SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); + SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pReq; cliMsg->st = taosGetTimestampUs(); + cliMsg->type = Normal; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; transSendAsync(thrd->asyncPool, &(cliMsg->q)); tsem_t* pSem = pCtx->pSem; tsem_wait(pSem); tsem_destroy(pSem); - free(pSem); + taosMemoryFree(pSem); } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 7123593a33bc5e64f28f9763ec4429e1531167fc..3078324c6bd611563e690f31eac246dc67436529 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -52,7 +52,7 @@ bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { return succ; } - char* buf = malloc(len + overhead + 8); // 8 extra bytes + char* buf = taosMemoryMalloc(len + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", len); *flen = len; @@ -78,7 +78,7 @@ bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { *flen = len; succ = false; } - free(buf); + taosMemoryFree(buf); return succ; } bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { @@ -92,15 +92,15 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { } void transConnCtxDestroy(STransConnCtx* ctx) { - free(ctx->ip); - free(ctx); + taosMemoryFree(ctx->ip); + taosMemoryFree(ctx); } void transFreeMsg(void* msg) { if (msg == NULL) { return; } - free((char*)msg - sizeof(STransMsgHead)); + taosMemoryFree((char*)msg - sizeof(STransMsgHead)); } int transInitBuffer(SConnBuffer* buf) { @@ -123,7 +123,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { SConnBuffer* p = connBuf; if (p->cap == 0) { - p->buf = (char*)calloc(CAPACITY, sizeof(char)); + p->buf = (char*)taosMemoryCalloc(CAPACITY, sizeof(char)); p->len = 0; p->cap = CAPACITY; p->total = -1; @@ -135,7 +135,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { uvBuf->len = CAPACITY - p->len; } else { p->cap = p->total; - p->buf = realloc(p->buf, p->cap); + p->buf = taosMemoryRealloc(p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; @@ -155,12 +155,12 @@ bool transReadComplete(SConnBuffer* connBuf) { } return false; } -int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {return 0;} +int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) { return 0; } -int transUnpackMsg(STransMsgHead* msgHead) {return 0;} +int transUnpackMsg(STransMsgHead* msgHead) { return 0; } int transDestroyBuffer(SConnBuffer* buf) { if (buf->cap > 0) { - tfree(buf->buf); + taosMemoryFreeClear(buf->buf); } transClearBuffer(buf); @@ -174,19 +174,19 @@ int transSetConnOption(uv_tcp_t* stream) { } SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { - SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); + SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); pool->index = 0; pool->nAsync = sz; - pool->asyncs = calloc(1, sizeof(uv_async_t) * pool->nAsync); + pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); for (int i = 0; i < pool->nAsync; i++) { uv_async_t* async = &(pool->asyncs[i]); uv_async_init(loop, async, cb); - SAsyncItem* item = calloc(1, sizeof(SAsyncItem)); + SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); item->pThrd = arg; QUEUE_INIT(&item->qmsg); - pthread_mutex_init(&item->mtx, NULL); + taosThreadMutexInit(&item->mtx, NULL); async->data = item; } @@ -197,11 +197,11 @@ void transDestroyAsyncPool(SAsyncPool* pool) { uv_async_t* async = &(pool->asyncs[i]); SAsyncItem* item = async->data; - pthread_mutex_destroy(&item->mtx); - free(item); + taosThreadMutexDestroy(&item->mtx); + taosMemoryFree(item); } - free(pool->asyncs); - free(pool); + taosMemoryFree(pool->asyncs); + taosMemoryFree(pool); } int transSendAsync(SAsyncPool* pool, queue* q) { int idx = pool->index; @@ -214,9 +214,9 @@ int transSendAsync(SAsyncPool* pool, queue* q) { SAsyncItem* item = async->data; int64_t st = taosGetTimestampUs(); - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_PUSH(&item->qmsg, q); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); int64_t el = taosGetTimestampUs() - st; if (el > 50) { // tInfo("lock and unlock cost: %d", (int)el); @@ -224,4 +224,142 @@ int transSendAsync(SAsyncPool* pool, queue* q) { return uv_async_send(async); } +void transCtxInit(STransCtx* ctx) { + // init transCtx + ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); +} +void transCtxCleanup(STransCtx* ctx) { + if (ctx->args == NULL) { + return; + } + + STransCtxVal* iter = taosHashIterate(ctx->args, NULL); + while (iter) { + iter->freeFunc(iter->val); + iter = taosHashIterate(ctx->args, iter); + } + + taosHashCleanup(ctx->args); + ctx->args = NULL; +} + +void transCtxMerge(STransCtx* dst, STransCtx* src) { + if (dst->args == NULL) { + dst->args = src->args; + dst->brokenVal = src->brokenVal; + src->args = NULL; + return; + } + void* key = NULL; + size_t klen = 0; + void* iter = taosHashIterate(src->args, NULL); + while (iter) { + STransCtxVal* sVal = (STransCtxVal*)iter; + key = taosHashGetKey(sVal, &klen); + + STransCtxVal* dVal = taosHashGet(dst->args, key, klen); + if (dVal) { + dVal->freeFunc(dVal->val); + } + taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); + iter = taosHashIterate(src->args, iter); + } + taosHashCleanup(src->args); +} +void* transCtxDumpVal(STransCtx* ctx, int32_t key) { + if (ctx->args == NULL) { + return NULL; + } + STransCtxVal* cVal = taosHashGet(ctx->args, (const void*)&key, sizeof(key)); + if (cVal == NULL) { + return NULL; + } + void* ret = NULL; + (*cVal->clone)(cVal->val, &ret); + return ret; +} +void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { + void* ret = NULL; + if (ctx->brokenVal.clone == NULL) { + return ret; + } + (*ctx->brokenVal.clone)(ctx->brokenVal.val, &ret); + + *msgType = ctx->brokenVal.msgType; + + return ret; +} + +void transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) { + queue->q = taosArrayInit(2, sizeof(void*)); + queue->freeFunc = freeFunc; +} +bool transQueuePush(STransQueue* queue, void* arg) { + if (queue->q == NULL) { + return true; + } + taosArrayPush(queue->q, &arg); + if (taosArrayGetSize(queue->q) > 1) { + return false; + } + return true; +} +void* transQueuePop(STransQueue* queue) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + return NULL; + } + void* ptr = taosArrayGetP(queue->q, 0); + taosArrayRemove(queue->q, 0); + return ptr; +} +int32_t transQueueSize(STransQueue* queue) { + if (queue->q == NULL) { + return 0; + } + return taosArrayGetSize(queue->q); +} +void* transQueueGet(STransQueue* queue, int i) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + return NULL; + } + if (i >= taosArrayGetSize(queue->q)) { + return NULL; + } + + void* ptr = taosArrayGetP(queue->q, i); + return ptr; +} + +void* transQueueRm(STransQueue* queue, int i) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + return NULL; + } + if (i >= taosArrayGetSize(queue->q)) { + return NULL; + } + void* ptr = taosArrayGetP(queue->q, i); + taosArrayRemove(queue->q, i); + return ptr; +} + +bool transQueueEmpty(STransQueue* queue) { + if (queue->q == NULL) { + return true; + } + return taosArrayGetSize(queue->q) == 0; +} +void transQueueClear(STransQueue* queue) { + if (queue->freeFunc != NULL) { + for (int i = 0; i < taosArrayGetSize(queue->q); i++) { + void* p = taosArrayGetP(queue->q, i); + queue->freeFunc(p); + } + } + taosArrayClear(queue->q); +} +void transQueueDestroy(STransQueue* queue) { + transQueueClear(queue); + taosArrayDestroy(queue->q); +} + #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 73d836319fcabd96fc42790b391fb39d8a6a154d..c137657a99e3d36ddbe0da0e862562fc77a7ece4 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -17,6 +17,12 @@ #include "transComm.h" +typedef struct { + int notifyCount; // + int init; // init or not + STransMsg msg; +} SSrvRegArg; + typedef struct SSrvConn { T_REF_DECLARE() uv_tcp_t* pTcp; @@ -31,10 +37,12 @@ typedef struct SSrvConn { void* pTransInst; // rpc init void* ahandle; // void* hostThrd; - SArray* srvMsgs; + STransQueue srvMsgs; - bool broken; // conn broken; + SSrvRegArg regArg; + bool broken; // conn broken; + ConnStatus status; struct sockaddr_in addr; struct sockaddr_in locaddr; @@ -47,20 +55,20 @@ typedef struct SSrvConn { } SSrvConn; typedef struct SSrvMsg { - SSrvConn* pConn; - STransMsg msg; - queue q; + SSrvConn* pConn; + STransMsg msg; + queue q; + STransMsgType type; } SSrvMsg; typedef struct SWorkThrdObj { - pthread_t thread; - uv_pipe_t* pipe; - uv_os_fd_t fd; - uv_loop_t* loop; - SAsyncPool* asyncPool; - - queue msg; - pthread_mutex_t msgMtx; + TdThread thread; + uv_pipe_t* pipe; + uv_os_fd_t fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + queue msg; + TdThreadMutex msgMtx; queue conn; void* pTransInst; @@ -68,7 +76,7 @@ typedef struct SWorkThrdObj { } SWorkThrdObj; typedef struct SServerObj { - pthread_t thread; + TdThread thread; uv_tcp_t server; uv_loop_t* loop; @@ -85,10 +93,26 @@ typedef struct SServerObj { static const char* notify = "a"; -// refactor later -static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen); - -static int uvAuthMsg(SSrvConn* pConn, char* msg, int msgLen); +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tTrace("server conn %p received release request", conn); \ + \ + STransMsg tmsg = {.code = 0, .handle = (void*)conn, .ahandle = NULL}; \ + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); \ + srvMsg->msg = tmsg; \ + srvMsg->type = Release; \ + srvMsg->pConn = conn; \ + if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ + return; \ + } \ + uvStartSendRespInternal(srvMsg); \ + return; \ + } \ + } while (0) static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -113,6 +137,13 @@ static void destroySmsg(SSrvMsg* smsg); static SSrvConn* createConn(void* hThrd); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); +static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd); +static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, + uvHandleRegister}; + static void uvDestroyConn(uv_handle_t* handle); // server and worker thread @@ -129,59 +160,6 @@ void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b transAllocBuffer(pBuf, buf); } -static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { - STransMsgHead* pHead = (STransMsgHead*)msg; - - int code = 0; - - if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { - // secured link, or no authentication - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, secured link, no auth is required", pConn->info); - return 0; - } - - if (!rpcIsReq(pHead->msgType)) { - // for response, if code is auth failure, it shall bypass the auth process - code = htonl(pHead->code); - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || - code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); - return 0; - } - } - - code = 0; - if (pHead->spi == pConn->spi) { - // authentication - SRpcDigest* pDigest = (SRpcDigest*)((char*)pHead + len - sizeof(SRpcDigest)); - - int32_t delta; - delta = (int32_t)htonl(pDigest->timeStamp); - delta -= (int32_t)taosGetTimestampSec(); - if (abs(delta) > 900) { - tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); - code = TSDB_CODE_RPC_INVALID_TIME_STAMP; - } else { - if (transAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { - // tDebug("%s, authentication failed, msg discarded", pConn->info); - code = TSDB_CODE_RPC_AUTH_FAILURE; - } else { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); - if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client - // tTrace("%s, message is authenticated", pConn->info); - } - } - } else { - tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); - code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; - } - - return code; -} - // refers specifically to query or insert timeout static void uvHandleActivityTimeout(uv_timer_t* handle) { SSrvConn* conn = handle->data; @@ -189,65 +167,60 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { } static void uvHandleReq(SSrvConn* pConn) { - SRecvInfo info; - SRecvInfo* p = &info; SConnBuffer* pBuf = &pConn->readBuf; - p->msg = pBuf->buf; - p->msgLen = pBuf->len; - p->ip = 0; - p->port = 0; - p->shandle = pConn->pTransInst; // - p->thandle = pConn; - p->chandle = NULL; - - STransMsgHead* pHead = (STransMsgHead*)p->msg; + char* msg = pBuf->buf; + uint32_t msgLen = pBuf->len; + + STransMsgHead* pHead = (STransMsgHead*)msg; if (pHead->secured == 1) { - STransUserMsg* uMsg = (STransUserMsg*)((char*)p->msg + p->msgLen - sizeof(STransUserMsg)); + STransUserMsg* uMsg = (STransUserMsg*)((char*)msg + msgLen - sizeof(STransUserMsg)); memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); } pHead->code = htonl(pHead->code); - - int32_t dlen = 0; - if (transDecompressMsg(NULL, 0, NULL)) { - // add compress later - // pHead = rpcDecompresSTransMsg(pHead); - } else { - pHead->msgLen = htonl(pHead->msgLen); - if (pHead->secured == 1) { - pHead->msgLen -= sizeof(STransUserMsg); - } - // + pHead->msgLen = htonl(pHead->msgLen); + if (pHead->secured == 1) { + pHead->msgLen -= sizeof(STransUserMsg); } + CONN_SHOULD_RELEASE(pConn, pHead); + STransMsg transMsg; transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = pHead->content; transMsg.msgType = pHead->msgType; transMsg.code = pHead->code; - transMsg.ahandle = NULL; + transMsg.ahandle = (void*)pHead->ahandle; transMsg.handle = NULL; transClearBuffer(&pConn->readBuf); pConn->inType = pHead->msgType; - - if (pHead->resflag == 0) { + if (pConn->status == ConnNormal) { + if (pHead->persist == 1) { + pConn->status = ConnAcquire; + transRefSrvHandle(pConn); + tDebug("server conn %p acquired by server app", pConn); + } + } + if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - transMsg.handle = pConn; tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); } else { - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, no resp ", pConn, + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); + taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen, pHead->noResp); + // no ref here + } + + if (pHead->noResp == 0) { + transMsg.handle = pConn; } - STrans* pTransInst = (STrans*)p->shandle; + STrans* pTransInst = pConn->pTransInst; (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); // uv_timer_start(&pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); - // auth - // validate msg type } void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { @@ -272,17 +245,19 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { tError("server conn %p read error: %s", conn, uv_err_name(nread)); if (nread < 0) { conn->broken = true; - uvNotifyLinkBrokenToApp(conn); - - // STrans* pTransInst = conn->pTransInst; - // if (pTransInst->efp != NULL && (pTransInst->efp)(NULL, conn->inType)) { - //} + if (conn->status == ConnAcquire) { + if (conn->regArg.init) { + STrans* pTransInst = conn->pTransInst; + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + } transUnrefSrvHandle(conn); } } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { buf->len = 2; - buf->base = calloc(1, sizeof(char) * buf->len); + buf->base = taosMemoryCalloc(1, sizeof(char) * buf->len); } void uvOnTimeoutCb(uv_timer_t* handle) { @@ -296,23 +271,40 @@ void uvOnSendCb(uv_write_t* req, int status) { transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); - if (conn->srvMsgs != NULL) { - assert(taosArrayGetSize(conn->srvMsgs) >= 1); - SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); - tTrace("server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs)); - taosArrayRemove(conn->srvMsgs, 0); + if (!transQueueEmpty(&conn->srvMsgs)) { + SSrvMsg* msg = transQueuePop(&conn->srvMsgs); + if (msg->type == Release && conn->status != ConnNormal) { + conn->status = ConnNormal; + transUnrefSrvHandle(conn); + } destroySmsg(msg); - // send second data, just use for push - if (taosArrayGetSize(conn->srvMsgs) > 0) { - tTrace("resent server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs)); - msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); - uvStartSendRespInternal(msg); + if (!transQueueEmpty(&conn->srvMsgs)) { + msg = (SSrvMsg*)transQueueGet(&conn->srvMsgs, 0); + if (msg->type == Register && conn->status == ConnAcquire) { + conn->regArg.notifyCount = 0; + conn->regArg.init = 1; + conn->regArg.msg = msg->msg; + if (conn->broken) { + STrans* pTransInst = conn->pTransInst; + (pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + transQueuePop(&conn->srvMsgs); + taosMemoryFree(msg); + + msg = (SSrvMsg*)transQueueGet(&conn->srvMsgs, 0); + if (msg != NULL) { + uvStartSendRespInternal(msg); + } + } else { + uvStartSendRespInternal(msg); + } } } } else { tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); - conn->broken = false; + conn->broken = true; transUnrefSrvHandle(conn); } } @@ -322,11 +314,10 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } else { tError("fail to dispatch conn to work thread"); } - free(req); + taosMemoryFree(req); } static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { - // impl later; tTrace("server conn %p prepare to send resp", smsg->pConn); SSrvConn* pConn = smsg->pConn; @@ -336,21 +327,29 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { pMsg->contLen = 0; } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + pHead->ahandle = (uint64_t)pMsg->ahandle; - pHead->secured = pMsg->code == 0 ? 1 : 0; // - pHead->msgType = smsg->pConn->inType + 1; + // pHead->secured = pMsg->code == 0 ? 1 : 0; // + if (!pConn->secured) { + pConn->secured = pMsg->code == 0 ? 1 : 0; + } + pHead->secured = pConn->secured; + + if (pConn->status == ConnNormal) { + pHead->msgType = pConn->inType + 1; + } else { + pHead->msgType = smsg->type == Release ? 0 : pMsg->msgType; + } + pHead->release = smsg->type == Release ? 1 : 0; pHead->code = htonl(pMsg->code); - // add more info + char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); - if (transCompressMsg(msg, len, NULL)) { - // impl later - } tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); - pHead->msgLen = htonl(len); + wb->base = msg; wb->len = len; } @@ -368,38 +367,27 @@ static void uvStartSendResp(SSrvMsg* smsg) { SSrvConn* pConn = smsg->pConn; if (pConn->broken == true) { + // persist by transUnrefSrvHandle(pConn); return; } - transUnrefSrvHandle(pConn); + if (pConn->status == ConnNormal) { + transUnrefSrvHandle(pConn); + } - if (taosArrayGetSize(pConn->srvMsgs) > 0) { - tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, taosInetNtoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); - taosArrayPush(pConn->srvMsgs, &smsg); + if (!transQueuePush(&pConn->srvMsgs, smsg)) { return; } - taosArrayPush(pConn->srvMsgs, &smsg); uvStartSendRespInternal(smsg); return; } -static void uvNotifyLinkBrokenToApp(SSrvConn* conn) { - STrans* pTransInst = conn->pTransInst; - if (pTransInst->efp != NULL && (*pTransInst->efp)(NULL, conn->inType) && T_REF_VAL_GET(conn) >= 2) { - STransMsg transMsg = {0}; - transMsg.msgType = conn->inType; - transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - // transRefSrvHandle(conn); - (*pTransInst->cfp)(pTransInst->parent, &transMsg, 0); - } -} static void destroySmsg(SSrvMsg* smsg) { if (smsg == NULL) { return; } transFreeMsg(smsg->msg.pCont); - free(smsg); + taosMemoryFree(smsg); } static void destroyAllConn(SWorkThrdObj* pThrd) { while (!QUEUE_IS_EMPTY(&pThrd->conn)) { @@ -408,6 +396,9 @@ static void destroyAllConn(SWorkThrdObj* pThrd) { QUEUE_INIT(h); SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue); + while (T_REF_VAL_GET(c) >= 2) { + transUnrefSrvHandle(c); + } transUnrefSrvHandle(c); } } @@ -418,9 +409,9 @@ void uvWorkerAsyncCb(uv_async_t* handle) { queue wq; // batch process to avoid to lock/unlock frequently - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_MOVE(&item->qmsg, &wq); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); while (!QUEUE_IS_EMPTY(&wq)) { queue* head = QUEUE_HEAD(&wq); @@ -431,20 +422,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) { tError("unexcept occurred, continue"); continue; } - if (msg->pConn == NULL) { - free(msg); - bool noConn = QUEUE_IS_EMPTY(&pThrd->conn); - if (noConn == true) { - uv_loop_close(pThrd->loop); - uv_stop(pThrd->loop); - } else { - destroyAllConn(pThrd); - // uv_loop_close(pThrd->loop); - pThrd->quit = true; - } - } else { - uvStartSendResp(msg); - } + (*transAsyncHandle[msg->type])(msg, pThrd); } } static void uvAcceptAsyncCb(uv_async_t* async) { @@ -459,7 +437,7 @@ static void uvShutDownCb(uv_shutdown_t* req, int status) { tDebug("conn failed to shut down: %s", uv_err_name(status)); } uv_close((uv_handle_t*)req->handle, uvDestroyConn); - free(req); + taosMemoryFree(req); } void uvOnAcceptCb(uv_stream_t* stream, int status) { @@ -468,11 +446,11 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { } SServerObj* pObj = container_of(stream, SServerObj, server); - uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_t* cli = (uv_tcp_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pObj->loop, cli); if (uv_accept(stream, (uv_stream_t*)cli) == 0) { - uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + uv_write_t* wr = (uv_write_t*)taosMemoryMalloc(sizeof(uv_write_t)); uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); @@ -482,7 +460,7 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); } else { uv_close((uv_handle_t*)cli, NULL); - free(cli); + taosMemoryFree(cli); } } void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { @@ -498,7 +476,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { // free memory allocated by assert(nread == strlen(notify)); assert(buf->base[0] == notify[0]); - free(buf->base); + taosMemoryFree(buf->base); SWorkThrdObj* pThrd = q->data; @@ -521,7 +499,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->hostThrd = pThrd; // init client handle - pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + pConn->pTcp = (uv_tcp_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, pConn->pTcp); pConn->pTcp->data = pConn; @@ -566,7 +544,7 @@ void* acceptThread(void* arg) { } static bool addHandleToWorkloop(void* arg) { SWorkThrdObj* pThrd = arg; - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (0 != uv_loop_init(pThrd->loop)) { return false; } @@ -577,12 +555,12 @@ static bool addHandleToWorkloop(void* arg) { pThrd->pipe->data = pThrd; QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); + taosThreadMutexInit(&pThrd->msgMtx, NULL); // conn set QUEUE_INIT(&pThrd->conn); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -598,7 +576,7 @@ static bool addHandleToAcceptloop(void* arg) { } // register an async here to quit server gracefully - srv->pAcceptAsync = calloc(1, sizeof(uv_async_t)); + srv->pAcceptAsync = taosMemoryCalloc(1, sizeof(uv_async_t)); uv_async_init(srv->loop, srv->pAcceptAsync, uvAcceptAsyncCb); srv->pAcceptAsync->data = srv; @@ -625,16 +603,19 @@ void* workerThread(void* arg) { static SSrvConn* createConn(void* hThrd) { SWorkThrdObj* pThrd = hThrd; - SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + SSrvConn* pConn = (SSrvConn*)taosMemoryCalloc(1, sizeof(SSrvConn)); QUEUE_INIT(&pConn->queue); QUEUE_PUSH(&pThrd->conn, &pConn->queue); - pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // - tTrace("conn %p created", pConn); + transQueueInit(&pConn->srvMsgs, NULL); + + memset(&pConn->regArg, 0, sizeof(pConn->regArg)); pConn->broken = false; + pConn->status = ConnNormal; transRefSrvHandle(pConn); + tTrace("server conn %p created", pConn); return pConn; } @@ -643,15 +624,9 @@ static void destroyConn(SSrvConn* conn, bool clear) { return; } transDestroyBuffer(&conn->readBuf); - - for (int i = 0; i < taosArrayGetSize(conn->srvMsgs); i++) { - SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i); - destroySmsg(msg); - } - conn->srvMsgs = taosArrayDestroy(conn->srvMsgs); if (clear) { - tTrace("try to destroy conn %p", conn); - uv_shutdown_t* req = malloc(sizeof(uv_shutdown_t)); + tTrace("server conn %p to be destroyed", conn); + uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } } @@ -664,52 +639,35 @@ static void uvDestroyConn(uv_handle_t* handle) { tDebug("server conn %p destroy", conn); uv_timer_stop(&conn->pTimer); + transQueueDestroy(&conn->srvMsgs); QUEUE_REMOVE(&conn->queue); - free(conn->pTcp); - // free(conn); + taosMemoryFree(conn->pTcp); + // taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { + tTrace("work thread quit"); uv_loop_close(thrd->loop); uv_stop(thrd->loop); } } -static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen) { - STransMsgHead* pHead = (STransMsgHead*)msg; - - if (pConn->spi && pConn->secured == 0) { - // add auth part - pHead->spi = pConn->spi; - STransDigestMsg* pDigest = (STransDigestMsg*)(msg + msgLen); - pDigest->timeStamp = htonl(taosGetTimestampSec()); - msgLen += sizeof(SRpcDigest); - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); - // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); - } else { - pHead->spi = 0; - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - } - - return msgLen; -} void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SServerObj* srv = calloc(1, sizeof(SServerObj)); - srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + SServerObj* srv = taosMemoryCalloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); srv->numOfThreads = numOfThreads; srv->workerIdx = 0; - srv->pThreadObj = (SWorkThrdObj**)calloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); - srv->pipe = (uv_pipe_t**)calloc(srv->numOfThreads, sizeof(uv_pipe_t*)); + srv->pThreadObj = (SWorkThrdObj**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); + srv->pipe = (uv_pipe_t**)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t*)); srv->ip = ip; srv->port = port; uv_loop_init(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); + SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); thrd->quit = false; srv->pThreadObj[i] = thrd; - srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; @@ -724,7 +682,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, if (false == addHandleToWorkloop(thrd)) { goto End; } - int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); + int err = taosThreadCreate(&(thrd->thread), NULL, workerThread, (void*)(thrd)); if (err == 0) { tDebug("sucess to create worker-thread %d", i); // printf("thread %d create\n", i); @@ -736,7 +694,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, if (false == addHandleToAcceptloop(srv)) { goto End; } - int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); + int err = taosThreadCreate(&srv->thread, NULL, acceptThread, (void*)srv); if (err == 0) { tDebug("success to create accept-thread"); } else { @@ -748,45 +706,95 @@ End: transCloseServer(srv); return NULL; } - +void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { + thrd->quit = true; + if (QUEUE_IS_EMPTY(&thrd->conn)) { + uv_loop_close(thrd->loop); + uv_stop(thrd->loop); + } else { + destroyAllConn(thrd); + } + taosMemoryFree(msg); +} +void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) { + // release handle to rpc init + SSrvConn* conn = msg->pConn; + if (conn->status == ConnAcquire) { + if (!transQueuePush(&conn->srvMsgs, msg)) { + return; + } + uvStartSendRespInternal(msg); + return; + } else if (conn->status == ConnRelease || conn->status == ConnNormal) { + tDebug("server conn %p already released, ignore release-msg", conn); + } + destroySmsg(msg); +} +void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd) { + // send msg to client + tDebug("server conn %p start to send resp (2/2)", msg->pConn); + uvStartSendResp(msg); +} +void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd) { + SSrvConn* conn = msg->pConn; + tDebug("server conn %p register brokenlink callback", conn); + if (conn->status == ConnAcquire) { + if (!transQueuePush(&conn->srvMsgs, msg)) { + return; + } + transQueuePop(&conn->srvMsgs); + conn->regArg.notifyCount = 0; + conn->regArg.init = 1; + conn->regArg.msg = msg->msg; + tDebug("server conn %p register brokenlink callback succ", conn); + + if (conn->broken) { + STrans* pTransInst = conn->pTransInst; + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + taosMemoryFree(msg); + } +} void destroyWorkThrd(SWorkThrdObj* pThrd) { if (pThrd == NULL) { return; } - pthread_join(pThrd->thread, NULL); - free(pThrd->loop); + taosThreadJoin(pThrd->thread, NULL); + taosMemoryFree(pThrd->loop); transDestroyAsyncPool(pThrd->asyncPool); - free(pThrd); + taosMemoryFree(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* msg = taosMemoryCalloc(1, sizeof(SSrvMsg)); + msg->type = Quit; tDebug("server send quit msg to work thread"); - - transSendAsync(pThrd->asyncPool, &srvMsg->q); + transSendAsync(pThrd->asyncPool, &msg->q); } void transCloseServer(void* arg) { // impl later SServerObj* srv = arg; + + tDebug("send quit msg to accept thread"); + uv_async_send(srv->pAcceptAsync); + taosThreadJoin(srv->thread, NULL); + for (int i = 0; i < srv->numOfThreads; i++) { sendQuitToWorkThrd(srv->pThreadObj[i]); destroyWorkThrd(srv->pThreadObj[i]); } - tDebug("send quit msg to accept thread"); - uv_async_send(srv->pAcceptAsync); - pthread_join(srv->thread, NULL); - - free(srv->pThreadObj); - free(srv->pAcceptAsync); - free(srv->loop); + taosMemoryFree(srv->pThreadObj); + taosMemoryFree(srv->pAcceptAsync); + taosMemoryFree(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - free(srv->pipe[i]); + taosMemoryFree(srv->pipe[i]); } - free(srv->pipe); + taosMemoryFree(srv->pipe); - free(srv); + taosMemoryFree(srv); } void transRefSrvHandle(void* handle) { @@ -804,17 +812,28 @@ void transUnrefSrvHandle(void* handle) { return; } int ref = T_REF_DEC((SSrvConn*)handle); - tDebug("handle %p ref count: %d", handle, ref); - + tDebug("server conn %p ref count: %d", handle, ref); if (ref == 0) { destroyConn((SSrvConn*)handle, true); } - // unref srv handle } void transReleaseSrvHandle(void* handle) { - // do nothing currently - // + if (handle == NULL) { + return; + } + SSrvConn* pConn = handle; + SWorkThrdObj* pThrd = pConn->hostThrd; + + STransMsg tmsg = {.code = 0, .handle = handle, .ahandle = NULL}; + + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); + srvMsg->msg = tmsg; + srvMsg->type = Release; + srvMsg->pConn = pConn; + + tTrace("server conn %p start to release", pConn); + transSendAsync(pThrd->asyncPool, &srvMsg->q); } void transSendResponse(const STransMsg* pMsg) { if (pMsg->handle == NULL) { @@ -822,11 +841,29 @@ void transSendResponse(const STransMsg* pMsg) { } SSrvConn* pConn = pMsg->handle; SWorkThrdObj* pThrd = pConn->hostThrd; + if (pThrd->quit) { + return; + } - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; - tTrace("server conn %p start to send resp", pConn); + srvMsg->type = Normal; + tTrace("server conn %p start to send resp (1/2)", pConn); + transSendAsync(pThrd->asyncPool, &srvMsg->q); +} +void transRegisterMsg(const STransMsg* msg) { + if (msg->handle == NULL) { + return; + } + SSrvConn* pConn = msg->handle; + SWorkThrdObj* pThrd = pConn->hostThrd; + + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); + srvMsg->pConn = pConn; + srvMsg->msg = *msg; + srvMsg->type = Register; + tTrace("server conn %p start to register brokenlink callback", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); } int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) { diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index b29bad07f08982c7644287d4ef9fa98cf10245bc..1245121d94376b2b5d677a7e918d2e20746449cf 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -7,11 +7,11 @@ add_executable(pushServer "") target_sources(transUT PRIVATE - "transUT.cc" + "transUT.cpp" ) target_sources(transportTest PRIVATE - "transportTests.cc" + "transportTests.cpp" ) target_sources (client PRIVATE diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index c6ff2480efa9c44a5ba9a6526c1934c67baf6cb8..7d3c3aa012465b5fac8879ebd9e480a74895778b 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -30,7 +30,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t * pOverSem; - pthread_t thread; + TdThread thread; void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -184,10 +184,10 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo) * appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < appThreads; ++i) { pInfo->index = i; @@ -196,7 +196,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 1fe13a35b1bd33e17f3b6a452d497caedf0e899c..e7fc7b45e1ed74b3a8d82983ba381757cadfc831 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -29,7 +29,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t *pOverSem; - pthread_t thread; + TdThread thread; void *pRpc; } SInfo; @@ -80,7 +80,7 @@ int main(int argc, char *argv[]) { char secret[TSDB_KEY_LEN] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.numOfEps = 1; @@ -161,10 +161,10 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo)*appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i=0; iindex = i; @@ -173,7 +173,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index b3c2f6ebdc6de3352259e52bf93322bfc2eb01ea..27a468a86fb7e3f011d5826ef524fe5bfb55aa6d 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -30,7 +30,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t * pOverSem; - pthread_t thread; + TdThread thread; void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -105,7 +105,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -184,10 +184,10 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo) * appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < appThreads; ++i) { pInfo->index = i; @@ -196,7 +196,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cpp similarity index 75% rename from source/libs/transport/test/transUT.cc rename to source/libs/transport/test/transUT.cpp index ec89d695a2bac264b4ee1f844c3baa7752c046f5..9dbebd6cfe3961227ed71678db58d0f248ae67f9 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cpp @@ -30,32 +30,13 @@ const char *ckey = "ckey"; class Server; int port = 7000; // server process - -static bool cliPersistHandle(void *parent, tmsg_t msgType) { - // client persist handle - return msgType == 2 || msgType == 4; -} - -typedef struct CbArgs { - tmsg_t msgType; -} CbArgs; - -static void *ConstructArgForSpecificMsgType(void *parent, tmsg_t msgType) { - if (msgType == 1 || msgType == 2) { - CbArgs *args = (CbArgs *)calloc(1, sizeof(CbArgs)); - args->msgType = msgType; - return args; - } - return NULL; -} // server except -static bool handleExcept(void *parent, tmsg_t msgType) { - // - return msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; -} + typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); // client process; static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); @@ -91,23 +72,6 @@ class Client { rpcClose(this->transCli); this->transCli = NULL; } - void SetPersistFP(bool (*pfp)(void *parent, tmsg_t msgType)) { - rpcClose(this->transCli); - rpcInit_.pfp = pfp; - this->transCli = rpcOpen(&rpcInit_); - } - void SetConstructFP(void *(*mfp)(void *parent, tmsg_t msgType)) { - rpcClose(this->transCli); - rpcInit_.mfp = mfp; - this->transCli = rpcOpen(&rpcInit_); - } - void SetPAndMFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) { - rpcClose(this->transCli); - - rpcInit_.pfp = pfp; - rpcInit_.mfp = mfp; - this->transCli = rpcOpen(&rpcInit_); - } void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { SEpSet epSet = {0}; @@ -126,7 +90,6 @@ class Client { SendAndRecv(req, resp); } - void SendWithHandle(SRpcMsg *req, SRpcMsg *resp) {} void SemWait() { tsem_wait(&this->sem); } void SemPost() { tsem_post(&this->sem); } void Reset() {} @@ -149,7 +112,6 @@ class Server { rpcInit_.label = (char *)label; rpcInit_.numOfThreads = 5; rpcInit_.cfp = processReq; - rpcInit_.efp = NULL; rpcInit_.user = (char *)user; rpcInit_.secret = (char *)secret; rpcInit_.ckey = (char *)ckey; @@ -160,17 +122,17 @@ class Server { this->transSrv = rpcOpen(&this->rpcInit_); taosMsleep(1000); } + void SetSrvContinueSend(CB cb) { + this->Stop(); + rpcInit_.cfp = cb; + this->Start(); + } void Stop() { if (this->transSrv == NULL) return; rpcClose(this->transSrv); this->transSrv = NULL; } - void SetExceptFp(bool (*efp)(void *parent, tmsg_t msgType)) { - this->Stop(); - rpcInit_.efp = efp; - this->Start(); - } - void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { + void SetSrvSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { this->Stop(); rpcInit_.cfp = cfp; this->Start(); @@ -198,9 +160,6 @@ static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { - for (int i = 0; i < 9; i++) { - rpcRefHandle(pMsg->handle, TAOS_CONN_SERVER); - } for (int i = 0; i < 10; i++) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = rpcMallocCont(100); @@ -210,6 +169,35 @@ static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { rpcSendResponse(&rpcMsg); } } +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + rpcReleaseHandle(pMsg->handle, TAOS_CONN_SERVER); +} +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + void *handle = pMsg->handle; + { + SRpcMsg rpcMsg1 = {0}; + rpcMsg1.pCont = rpcMallocCont(100); + rpcMsg1.contLen = 100; + rpcMsg1.handle = handle; + rpcMsg1.code = 0; + rpcRegisterBrokenLinkArg(&rpcMsg1); + } + taosMsleep(10); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); +} // client process; static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { Client *client = (Client *)parent; @@ -262,29 +250,13 @@ class TransObj { // srv->Stop(); } - void SetCliPersistFp(bool (*pfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->SetPersistFP(pfp); - } - void SetCliMFp(void *(*mfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->SetConstructFP(mfp); - } - void SetCliMAndPFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->SetPAndMFp(pfp, mfp); - } // call when link broken, and notify query or fetch stop - void SetSrvExceptFp(bool (*efp)(void *parent, tmsg_t msgType)) { - //////// - srv->SetExceptFp(efp); - } void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { /////// srv->SetSrvContinueSend(cfp); } void RestartSrv() { srv->Restart(); } - void cliStop() { + void StopCli() { /////// cli->Stop(); } @@ -315,7 +287,7 @@ class TransEnv : public ::testing::Test { }; TEST_F(TransEnv, 01sendAndRec) { - for (int i = 0; i < 1; i++) { + for (int i = 0; i < 10; i++) { SRpcMsg req = {0}, resp = {0}; req.msgType = 0; req.pCont = rpcMallocCont(10); @@ -358,60 +330,74 @@ TEST_F(TransEnv, clientUserDefined) { } TEST_F(TransEnv, cliPersistHandle) { - tr->SetCliPersistFp(cliPersistHandle); SRpcMsg resp = {0}; + void * handle = NULL; for (int i = 0; i < 10; i++) { - SRpcMsg req = {.handle = resp.handle, .noResp = 0}; + SRpcMsg req = {0}; + req.handle = resp.handle; + req.persistHandle = 1; + + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + // if (i == 5) { + // std::cout << "stop server" << std::endl; + // tr->StopSrv(); + //} + // if (i >= 6) { + // EXPECT_TRUE(resp.code != 0); + //} + handle = resp.handle; + } + rpcReleaseHandle(handle, TAOS_CONN_CLIENT); + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; tr->cliSendAndRecv(&req, &resp); - if (i == 5) { - std::cout << "stop server" << std::endl; - tr->StopSrv(); - } - if (i >= 6) { - EXPECT_TRUE(resp.code != 0); - } } + + taosMsleep(1000); ////////////////// } -TEST_F(TransEnv, cliReleaseHandle) { - tr->SetCliPersistFp(cliPersistHandle); - +TEST_F(TransEnv, srvReleaseHandle) { SRpcMsg resp = {0}; - for (int i = 0; i < 10; i++) { - SRpcMsg req = {.handle = resp.handle}; + tr->SetSrvContinueSend(processReleaseHandleCb); + // tr->Restart(processReleaseHandleCb); + void * handle = NULL; + SRpcMsg req = {0}; + for (int i = 0; i < 1; i++) { + memset(&req, 0, sizeof(req)); + req.handle = resp.handle; + req.persistHandle = 1; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; - tr->cliSendAndRecvNoHandle(&req, &resp); - // if (i == 5) { - // std::cout << "stop server" << std::endl; - // tr->StopSrv(); - //} - // if (i >= 6) { + tr->cliSendAndRecv(&req, &resp); + // tr->cliSendAndRecvNoHandle(&req, &resp); EXPECT_TRUE(resp.code == 0); - //} } ////////////////// } TEST_F(TransEnv, cliReleaseHandleExcept) { - tr->SetCliPersistFp(cliPersistHandle); - SRpcMsg resp = {0}; - for (int i = 0; i < 10; i++) { - SRpcMsg req = {.handle = resp.handle}; + SRpcMsg req = {0}; + for (int i = 0; i < 3; i++) { + memset(&req, 0, sizeof(req)); + req.handle = resp.handle; + req.persistHandle = 1; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; - tr->cliSendAndRecvNoHandle(&req, &resp); - if (i == 5) { + tr->cliSendAndRecv(&req, &resp); + if (i == 1) { std::cout << "stop server" << std::endl; tr->StopSrv(); } - if (i >= 6) { + if (i > 1) { EXPECT_TRUE(resp.code != 0); } } @@ -419,28 +405,32 @@ TEST_F(TransEnv, cliReleaseHandleExcept) { } TEST_F(TransEnv, srvContinueSend) { tr->SetSrvContinueSend(processContinueSend); + SRpcMsg req = {0}, resp = {0}; for (int i = 0; i < 10; i++) { - SRpcMsg req = {0}, resp = {0}; + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; tr->cliSendAndRecv(&req, &resp); } - taosMsleep(2000); + taosMsleep(1000); } TEST_F(TransEnv, srvPersistHandleExcept) { tr->SetSrvContinueSend(processContinueSend); - tr->SetCliPersistFp(cliPersistHandle); + // tr->SetCliPersistFp(cliPersistHandle); SRpcMsg resp = {0}; + SRpcMsg req = {0}; for (int i = 0; i < 5; i++) { - SRpcMsg req = {.handle = resp.handle}; + memset(&req, 0, sizeof(req)); + req.handle = resp.handle; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; tr->cliSendAndRecv(&req, &resp); if (i > 2) { - tr->cliStop(); + tr->StopCli(); break; } } @@ -450,10 +440,11 @@ TEST_F(TransEnv, srvPersistHandleExcept) { } TEST_F(TransEnv, cliPersistHandleExcept) { tr->SetSrvContinueSend(processContinueSend); - tr->SetCliPersistFp(cliPersistHandle); SRpcMsg resp = {0}; + SRpcMsg req = {0}; for (int i = 0; i < 5; i++) { - SRpcMsg req = {.handle = resp.handle}; + memset(&req, 0, sizeof(req)); + req.handle = resp.handle; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; @@ -472,14 +463,31 @@ TEST_F(TransEnv, multiCliPersistHandleExcept) { // conn broken } TEST_F(TransEnv, queryExcept) { - tr->SetSrvExceptFp(handleExcept); - - // query and conn is broken + tr->SetSrvContinueSend(processRegisterFailure); + SRpcMsg resp = {0}; + SRpcMsg req = {0}; + for (int i = 0; i < 5; i++) { + memset(&req, 0, sizeof(req)); + req.handle = resp.handle; + req.persistHandle = 1; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + if (i == 2) { + rpcReleaseHandle(resp.handle, TAOS_CONN_CLIENT); + tr->StopCli(); + break; + } + } + taosMsleep(4 * 1000); } TEST_F(TransEnv, noResp) { SRpcMsg resp = {0}; + SRpcMsg req = {0}; for (int i = 0; i < 5; i++) { - SRpcMsg req = {.noResp = 1}; + memset(&req, 0, sizeof(req)); + req.noResp = 1; req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; diff --git a/source/libs/transport/test/transportTests.cc b/source/libs/transport/test/transportTests.cc deleted file mode 100644 index 53910aa30c1d7bcea1ce0d6dc46f1bb579c3c3c1..0000000000000000000000000000000000000000 --- a/source/libs/transport/test/transportTests.cc +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifdef USE_UV - -#include -#include -#include -#include -#include -#include - -#include "transComm.h" -#include "transportInt.h" -#include "trpc.h" - -using namespace std; - -struct QueueElem { - queue q; - int val; -}; -class QueueObj { - public: - QueueObj() { - // avoid formate - QUEUE_INIT(&head); - } - void Push(QueueElem *el) { - // avoid formate - QUEUE_PUSH(&head, &el->q); - } - QueueElem *Pop() { - QueueElem *el = NULL; - if (!IsEmpty()) { - queue *h = QUEUE_HEAD(&head); - el = QUEUE_DATA(h, QueueElem, q); - QUEUE_REMOVE(h); - } - return el; - } - bool IsEmpty() { - // avoid formate - return QUEUE_IS_EMPTY(&head); - } - void RmElem(QueueElem *el) { - // impl - QUEUE_REMOVE(&el->q); - } - void ForEach(std::vector &result) { - queue *h; - QUEUE_FOREACH(h, &head) { - // add more - QueueElem *el = QUEUE_DATA(h, QueueElem, q); - result.push_back(el->val); - } - } - - private: - queue head; -}; - -class QueueEnv : public ::testing::Test { - protected: - virtual void SetUp() { - // TODO - q = new QueueObj(); - } - virtual void TearDown() { - delete q; - // formate - } - QueueObj *q; -}; - -TEST_F(QueueEnv, testPushAndPop) { - // add more test - assert(q->IsEmpty()); - - for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); - el->val = i; - q->Push(el); - } - int i = 0; - while (!q->IsEmpty()) { - QueueElem *el = q->Pop(); - assert(el->val == i++); - free(el); - } - assert(q->IsEmpty()); -} -TEST_F(QueueEnv, testRm) { - // add more test - - std::vector set; - assert(q->IsEmpty()); - - for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); - el->val = i; - q->Push(el); - set.push_back(el); - } - for (int i = set.size() - 1; i >= 0; i--) { - QueueElem *el = set[i]; - q->RmElem(el); - free(el); - } - assert(q->IsEmpty()); -} -TEST_F(QueueEnv, testIter) { - // add more test - assert(q->IsEmpty()); - std::vector vals; - for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); - el->val = i; - q->Push(el); - vals.push_back(i); - } - std::vector result; - q->ForEach(result); - assert(result.size() == vals.size()); -} - -#endif diff --git a/source/libs/transport/test/transportTests.cpp b/source/libs/transport/test/transportTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4165e6d8c9a1bc16faa26a59f76c1b47eebccb59 --- /dev/null +++ b/source/libs/transport/test/transportTests.cpp @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef USE_UV + +#include +#include +#include +#include +#include +#include + +#include "transComm.h" +#include "transportInt.h" +#include "trpc.h" + +using namespace std; + +struct QueueElem { + queue q; + int val; +}; +class QueueObj { + public: + QueueObj() { + // avoid formate + QUEUE_INIT(&head); + } + void Push(QueueElem *el) { + // avoid formate + QUEUE_PUSH(&head, &el->q); + } + QueueElem *Pop() { + QueueElem *el = NULL; + if (!IsEmpty()) { + queue *h = QUEUE_HEAD(&head); + el = QUEUE_DATA(h, QueueElem, q); + QUEUE_REMOVE(h); + } + return el; + } + bool IsEmpty() { + // avoid formate + return QUEUE_IS_EMPTY(&head); + } + void RmElem(QueueElem *el) { + // impl + QUEUE_REMOVE(&el->q); + } + void ForEach(std::vector &result) { + queue *h; + QUEUE_FOREACH(h, &head) { + // add more + QueueElem *el = QUEUE_DATA(h, QueueElem, q); + result.push_back(el->val); + } + } + + private: + queue head; +}; + +class QueueEnv : public ::testing::Test { + protected: + virtual void SetUp() { + // TODO + q = new QueueObj(); + } + virtual void TearDown() { + delete q; + // formate + } + QueueObj *q; +}; + +TEST_F(QueueEnv, testPushAndPop) { + // add more test + assert(q->IsEmpty()); + + for (int i = 0; i < 100; i++) { + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); + el->val = i; + q->Push(el); + } + int i = 0; + while (!q->IsEmpty()) { + QueueElem *el = q->Pop(); + assert(el->val == i++); + taosMemoryFree(el); + } + assert(q->IsEmpty()); +} +TEST_F(QueueEnv, testRm) { + // add more test + + std::vector set; + assert(q->IsEmpty()); + + for (int i = 0; i < 100; i++) { + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); + el->val = i; + q->Push(el); + set.push_back(el); + } + for (int i = set.size() - 1; i >= 0; i--) { + QueueElem *el = set[i]; + q->RmElem(el); + taosMemoryFree(el); + } + assert(q->IsEmpty()); +} +TEST_F(QueueEnv, testIter) { + // add more test + assert(q->IsEmpty()); + std::vector vals; + for (int i = 0; i < 100; i++) { + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); + el->val = i; + q->Push(el); + vals.push_back(i); + } + std::vector result; + q->ForEach(result); + assert(result.size() == vals.size()); +} + +class TransCtxEnv : public ::testing::Test { + protected: + virtual void SetUp() { + ctx = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); + transCtxInit(ctx); + // TODO + } + virtual void TearDown() { + transCtxCleanup(ctx); + // formate + } + STransCtx *ctx; +}; + +TEST_F(TransCtxEnv, mergeTest) { + int key = 1; + { + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + taosMemoryFree(src); + } + EXPECT_EQ(2, taosHashGetSize(ctx->args)); + { + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + taosMemoryFree(src); + } + std::string val("Hello"); + EXPECT_EQ(4, taosHashGetSize(ctx->args)); + { + key = 1; + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryCalloc(1, 11); + memcpy(val1.val, val.c_str(), val.size()); + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryCalloc(1, 11); + memcpy(val1.val, val.c_str(), val.size()); + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + taosMemoryFree(src); + } + EXPECT_EQ(4, taosHashGetSize(ctx->args)); + + char *skey = (char *)transCtxDumpVal(ctx, 1); + EXPECT_EQ(0, strcmp(skey, val.c_str())); + taosMemoryFree(skey); + + skey = (char *)transCtxDumpVal(ctx, 2); + EXPECT_EQ(0, strcmp(skey, val.c_str())); +} +#endif diff --git a/source/libs/transport/test/uv.c b/source/libs/transport/test/uv.c index 4c7d30900b220c5b1ea87fb55ffb16a415541986..fb026ef1a61fc190eb1b560520a83809191023d5 100644 --- a/source/libs/transport/test/uv.c +++ b/source/libs/transport/test/uv.c @@ -1,17 +1,16 @@ #include -#include +#include #include #include #include #include "task.h" -#include #define NUM_OF_THREAD 1 #define TIMEOUT 10000 typedef struct SThreadObj { - pthread_t thread; + TdThread thread; uv_pipe_t *pipe; uv_loop_t *loop; uv_async_t *workerAsync; // @@ -39,7 +38,7 @@ void echo_write(uv_write_t *req, int status) { fprintf(stderr, "Write error %s\n", uv_err_name(status)); } printf("write data to client\n"); - free(req); + taosMemoryFree(req); } void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { @@ -48,14 +47,14 @@ void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { pConn->ref += 1; printf("read data %d\n", nread, buf->base, buf->len); if (nread > 0) { - uv_write_t *req = (uv_write_t *)malloc(sizeof(uv_write_t)); + uv_write_t *req = (uv_write_t *)taosMemoryMalloc(sizeof(uv_write_t)); // dispatch request to database other process thread // just write out uv_buf_t write_out; write_out.base = buf->base; write_out.len = nread; uv_write((uv_write_t *)req, client, &write_out, 1, echo_write); - free(buf->base); + taosMemoryFree(buf->base); return; } @@ -64,11 +63,11 @@ void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { fprintf(stderr, "Read error %s\n", uv_err_name(nread)); uv_close((uv_handle_t *)client, NULL); } - free(buf->base); + taosMemoryFree(buf->base); } void alloc_buffer(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { - buf->base = malloc(suggested_size); + buf->base = taosMemoryMalloc(suggested_size); buf->len = suggested_size; } @@ -80,10 +79,10 @@ void on_new_connection(uv_stream_t *s, int status) { SServerObj *pObj = container_of(s, SServerObj, server); printf("new_connection from client\n"); - uv_tcp_t *client = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + uv_tcp_t *client = (uv_tcp_t *)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pObj->loop, client); if (uv_accept(s, (uv_stream_t *)client) == 0) { - uv_write_t *write_req = (uv_write_t *)malloc(sizeof(uv_write_t)); + uv_write_t *write_req = (uv_write_t *)taosMemoryMalloc(sizeof(uv_write_t)); uv_buf_t dummy_buf = uv_buf_init("a", 1); // despatch to worker thread pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; @@ -113,13 +112,13 @@ void child_on_new_connection(uv_stream_t *q, ssize_t nread, uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SConnCtx *pConn = malloc(sizeof(SConnCtx)); + SConnCtx *pConn = taosMemoryMalloc(sizeof(SConnCtx)); /* init conn timer*/ - pConn->pTimer = malloc(sizeof(uv_timer_t)); + pConn->pTimer = taosMemoryMalloc(sizeof(uv_timer_t)); uv_timer_init(pObj->loop, pConn->pTimer); - pConn->pClient = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + pConn->pClient = (uv_tcp_t *)taosMemoryMalloc(sizeof(uv_tcp_t)); pConn->pWorkerAsync = pObj->workerAsync; // thread safty uv_tcp_init(pObj->loop, pConn->pClient); @@ -131,10 +130,10 @@ void child_on_new_connection(uv_stream_t *q, ssize_t nread, uv_read_start((uv_stream_t *)(pConn->pClient), alloc_buffer, echo_read); } else { uv_timer_stop(pConn->pTimer); - free(pConn->pTimer); + taosMemoryFree(pConn->pTimer); uv_close((uv_handle_t *)pConn->pClient, NULL); - free(pConn->pClient); - free(pConn); + taosMemoryFree(pConn->pClient); + taosMemoryFree(pConn); } } @@ -145,13 +144,13 @@ static void workerAsyncCallback(uv_async_t *handle) { void *worker_thread(void *arg) { SThreadObj *pObj = (SThreadObj *)arg; int fd = pObj->fd; - pObj->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + pObj->loop = (uv_loop_t *)taosMemoryMalloc(sizeof(uv_loop_t)); uv_loop_init(pObj->loop); uv_pipe_init(pObj->loop, pObj->pipe, 1); uv_pipe_open(pObj->pipe, fd); - pObj->workerAsync = malloc(sizeof(uv_async_t)); + pObj->workerAsync = taosMemoryMalloc(sizeof(uv_async_t)); uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCallback); uv_read_start((uv_stream_t *)pObj->pipe, alloc_buffer, child_on_new_connection); @@ -160,19 +159,19 @@ void *worker_thread(void *arg) { } int main() { - SServerObj *server = calloc(1, sizeof(SServerObj)); - server->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + SServerObj *server = taosMemoryCalloc(1, sizeof(SServerObj)); + server->loop = (uv_loop_t *)taosMemoryMalloc(sizeof(uv_loop_t)); server->numOfThread = NUM_OF_THREAD; server->workerIdx = 0; server->pThreadObj = - (SThreadObj **)calloc(server->numOfThread, sizeof(SThreadObj *)); - server->pipe = (uv_pipe_t **)calloc(server->numOfThread, sizeof(uv_pipe_t *)); + (SThreadObj **)taosMemoryCalloc(server->numOfThread, sizeof(SThreadObj *)); + server->pipe = (uv_pipe_t **)taosMemoryCalloc(server->numOfThread, sizeof(uv_pipe_t *)); uv_loop_init(server->loop); for (int i = 0; i < server->numOfThread; i++) { - server->pThreadObj[i] = (SThreadObj *)calloc(1, sizeof(SThreadObj)); - server->pipe[i] = (uv_pipe_t *)calloc(2, sizeof(uv_pipe_t)); + server->pThreadObj[i] = (SThreadObj *)taosMemoryCalloc(1, sizeof(SThreadObj)); + server->pipe[i] = (uv_pipe_t *)taosMemoryCalloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { @@ -183,7 +182,7 @@ int main() { server->pThreadObj[i]->fd = fds[0]; server->pThreadObj[i]->pipe = &(server->pipe[i][1]); // init read - int err = pthread_create(&(server->pThreadObj[i]->thread), NULL, + int err = taosThreadCreate(&(server->pThreadObj[i]->thread), NULL, worker_thread, (void *)(server->pThreadObj[i])); if (err == 0) { printf("thread %d create\n", i); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 70ec7e06557d75358f2b4e33a8aec06b493e06d2..cbe7cb81db9f855597bc717e6bfe441b281bcc11 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -74,7 +74,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { uint64_t magic = WAL_MAGIC; - char* buf = malloc(readSize + 5); + char* buf = taosMemoryMalloc(readSize + 5); if (buf == NULL) { taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; @@ -83,7 +83,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { taosLSeekFile(pFile, -readSize, SEEK_END); if (readSize != taosReadFile(pFile, buf, readSize)) { - free(buf); + taosMemoryFree(buf); taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -104,7 +104,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { SWalHead *logContent = (SWalHead*)found; if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted - free(buf); + taosMemoryFree(buf); taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -215,7 +215,7 @@ int walRollFileInfo(SWal* pWal) { } // TODO: change to emplace back - SWalFileInfo* pNewInfo = malloc(sizeof(SWalFileInfo)); + SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo)); if (pNewInfo == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -226,7 +226,7 @@ int walRollFileInfo(SWal* pWal) { pNewInfo->closeTs = -1; pNewInfo->fileSize = 0; taosArrayPush(pArray, pNewInfo); - free(pNewInfo); + taosMemoryFree(pNewInfo); return 0; } @@ -378,7 +378,7 @@ int walSaveMeta(SWal* pWal) { walBuildMetaName(pWal, metaVer, fnameStr); taosRemoveFile(fnameStr); } - free(serialized); + taosMemoryFree(serialized); return 0; } @@ -395,7 +395,7 @@ int walLoadMeta(SWal* pWal) { int64_t file_size = 0; taosStatFile(fnameStr, &file_size, NULL); int size = (int)file_size; - char* buf = malloc(size + 5); + char* buf = taosMemoryMalloc(size + 5); if (buf == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -409,12 +409,12 @@ int walLoadMeta(SWal* pWal) { if (taosReadFile(pFile, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - free(buf); + taosMemoryFree(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); taosCloseFile(&pFile); - free(buf); + taosMemoryFree(buf); return code; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index b1caeab3b1396ae601326217c956133d776d54f8..cf40c998deabb1b869b8e64cbd84b7f52dbb01ea 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -25,7 +25,7 @@ typedef struct { int8_t inited; uint32_t seq; int32_t refSetId; - pthread_t thread; + TdThread thread; } SWalMgmt; static SWalMgmt tsWal = {0, .seq = 1}; @@ -63,7 +63,7 @@ void walCleanUp() { } SWal *walOpen(const char *path, SWalCfg *pCfg) { - SWal *pWal = malloc(sizeof(SWal)); + SWal *pWal = taosMemoryMalloc(sizeof(SWal)); if (pWal == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -88,7 +88,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -101,17 +101,17 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.headVer = WAL_HEAD_VER; pWal->writeHead.magic = WAL_MAGIC; - if (pthread_mutex_init(&pWal->mutex, NULL) < 0) { + if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } pWal->refId = taosAddRef(tsWal.refSetId, pWal); if (pWal->refId < 0) { - pthread_mutex_destroy(&pWal->mutex); + taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -119,9 +119,9 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (walCheckAndRepairMeta(pWal) < 0) { taosRemoveRef(tsWal.refSetId, pWal->refId); - pthread_mutex_destroy(&pWal->mutex); + taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -156,7 +156,7 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { } void walClose(SWal *pWal) { - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); taosCloseFile(&pWal->pWriteLogTFile); pWal->pWriteLogTFile = NULL; taosCloseFile(&pWal->pWriteIdxTFile); @@ -164,7 +164,7 @@ void walClose(SWal *pWal) { walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } @@ -173,8 +173,8 @@ static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); - pthread_mutex_destroy(&pWal->mutex); - tfree(pWal); + taosThreadMutexDestroy(&pWal->mutex); + taosMemoryFreeClear(pWal); } static bool walNeedFsync(SWal *pWal) { @@ -223,17 +223,17 @@ static void *walThreadFunc(void *param) { } static int32_t walCreateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { + if (taosThreadCreate(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { wError("failed to create wal thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread)); return 0; @@ -243,7 +243,7 @@ static void walStopThread() { atomic_store_8(&tsWal.stop, 1); if (taosCheckPthreadValid(tsWal.thread)) { - pthread_join(tsWal.thread, NULL); + taosThreadJoin(tsWal.thread, NULL); } wDebug("wal thread is stopped"); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 8d3acef9e74646629284d96a783847088525a1f1..e15c1620480c1825714ab7d36550b31a5be9132f 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -17,7 +17,7 @@ #include "taoserror.h" SWalReadHandle *walOpenReadHandle(SWal *pWal) { - SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); + SWalReadHandle *pRead = taosMemoryMalloc(sizeof(SWalReadHandle)); if (pRead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -30,10 +30,10 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { pRead->curFileFirstVer = -1; pRead->capacity = 0; pRead->status = 0; - pRead->pHead = malloc(sizeof(SWalHead)); + pRead->pHead = taosMemoryMalloc(sizeof(SWalHead)); if (pRead->pHead == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - free(pRead); + taosMemoryFree(pRead); return NULL; } return pRead; @@ -42,8 +42,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { void walCloseReadHandle(SWalReadHandle *pRead) { taosCloseFile(&pRead->pReadIdxTFile); taosCloseFile(&pRead->pReadLogTFile); - tfree(pRead->pHead); - free(pRead); + taosMemoryFreeClear(pRead->pHead); + taosMemoryFree(pRead); } int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } @@ -156,7 +156,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return -1; } if (pRead->capacity < pRead->pHead->head.len) { - void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); + void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); if (ptr == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -195,7 +195,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { return code; } if (*ppHead == NULL) { - void *ptr = realloc(*ppHead, sizeof(SWalHead)); + void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead)); if (ptr == NULL) { return -1; } @@ -208,9 +208,9 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { if (walValidHeadCksum(*ppHead) != 0) { return -1; } - void *ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); + void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); if (ptr == NULL) { - free(*ppHead); + taosMemoryFree(*ppHead); *ppHead = NULL; return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 0f6ac0b214741eab7f365603457479ff778cf876..a578c6f3685c2cfdce8870b04daa5508ab411c29 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -38,7 +38,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); // find correct file if (ver < walGetLastFileFirstVer(pWal)) { @@ -65,20 +65,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // TODO:change to deserialize function if (pIdxTFile == NULL) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code < 0) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } // read idx file and get log file pos // TODO:change to deserialize function SWalIdxEntry entry; if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } ASSERT(entry.ver == ver); @@ -87,13 +87,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); if (pLogTFile == NULL) { // TODO - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { // TODO - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } // validate offset @@ -127,7 +127,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; // unlock - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return 0; } @@ -253,7 +253,8 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { return 0; } -int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) { +int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body, + int32_t bodyLen) { if (pWal == NULL) return -1; int code = 0; @@ -283,7 +284,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in ASSERT(pWal->writeCur >= 0); - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { walSetWrite(pWal); @@ -296,6 +297,10 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in int64_t offset = walGetCurFileOffset(pWal); pWal->writeHead.head.len = bodyLen; pWal->writeHead.head.msgType = msgType; + + // sync info + pWal->writeHead.head.syncMeta = syncMeta; + pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); @@ -327,11 +332,20 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return 0; } +int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) { + SSyncLogMeta syncMeta = { + .isWeek = -1, + .seqNum = UINT64_MAX, + .term = UINT64_MAX, + }; + return walWriteWithSyncInfo(pWal, index, msgType, syncMeta, body, bodyLen); +} + void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 69ae83154acc7bbdbedc1f05654bbf8993c5c5e7..b1de3a1dcee592d5e69dd40504008f9a100647bc 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -19,7 +19,7 @@ class WalCleanEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; @@ -27,7 +27,7 @@ class WalCleanEnv : public ::testing::Test { pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -51,13 +51,13 @@ class WalCleanDeleteEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->retentionPeriod = 0; pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -86,7 +86,7 @@ class WalKeepEnv : public ::testing::Test { } void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; @@ -94,7 +94,7 @@ class WalKeepEnv : public ::testing::Test { pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -172,7 +172,7 @@ TEST_F(WalCleanEnv, serialize) { ASSERT(code == 0); char* ss = walMetaSerialize(pWal); printf("%s\n", ss); - free(ss); + taosMemoryFree(ss); code = walSaveMeta(pWal); ASSERT(code == 0); } @@ -216,8 +216,8 @@ TEST_F(WalKeepEnv, readOldMeta) { for (int i = 0; i < len; i++) { EXPECT_EQ(oldss[i], newss[i]); } - free(oldss); - free(newss); + taosMemoryFree(oldss); + taosMemoryFree(newss); } TEST_F(WalCleanEnv, write) { diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 6429cdc9d3d5dea6d1418994aadabcf2db3dfeb0..8b452947d393c307e3ddfe2883c906583e5a5677 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -13,25 +13,18 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) // add char interlocked_add_fetch_8(char volatile* ptr, char val) { -#ifdef _TD_GO_DLL_ - return __sync_fetch_and_add(ptr, val) + val; -#else return _InterlockedExchangeAdd8(ptr, val) + val; -#endif } short interlocked_add_fetch_16(short volatile* ptr, short val) { -#ifdef _TD_GO_DLL_ - return __sync_fetch_and_add(ptr, val) + val; -#else return _InterlockedExchangeAdd16(ptr, val) + val; -#endif } long interlocked_add_fetch_32(long volatile* ptr, long val) { @@ -39,11 +32,7 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) { } __int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { -//#ifdef _WIN64 return InterlockedExchangeAdd64(ptr, val) + val; -//#else -// return _InterlockedExchangeAdd(ptr, val) + val; -//#endif } char interlocked_and_fetch_8(char volatile* ptr, char val) { @@ -197,3 +186,709 @@ int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { return old; } #endif + + +int8_t atomic_load_8(int8_t volatile *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(int8_t volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((ptr), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_load_16(int16_t volatile *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(int16_t volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((ptr), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_load_32(int32_t volatile *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(int32_t volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((ptr), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_load_64(int64_t volatile *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(int64_t volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((ptr), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_load_ptr(void *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(void* volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((void**)(ptr), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(int8_t volatile*)(ptr)) = (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(int16_t volatile*)(ptr)) = (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(int32_t volatile*)(ptr)) = (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(int64_t volatile*)(ptr)) = (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(void* volatile*)(ptr)) = (void*)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchange8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val); +#else + return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchange16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val); +#else + return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchange((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val); +#else + return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchange64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val); +#else + return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_exchange_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + #ifdef _WIN64 + return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); + #else + return _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); + #endif +#elif defined(_TD_NINGSI_60) + return atomic_exchange_ptr_impl((void *)ptr, (void*)val); +#else + return __atomic_exchange_n((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchange8((int8_t volatile*)(ptr), (int8_t)(newval), (int8_t)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap(ptr, oldval, newval); +#endif +} + +int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchange16((int16_t volatile*)(ptr), (int16_t)(newval), (int16_t)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap(ptr, oldval, newval); +#endif +} + +int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchange((int32_t volatile*)(ptr), (int32_t)(newval), (int32_t)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap(ptr, oldval, newval); +#endif +} + +int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchange64((int64_t volatile*)(ptr), (int64_t)(newval), (int64_t)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap(ptr, oldval, newval); +#endif +} + +void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap((void **)ptr, oldval, newval); +#endif +} + +int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_add_fetch_ptr(void *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_add_ptr(void *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAddptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_8((int8_t volatile*)(ptr), -(int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_16((int16_t volatile*)(ptr), -(int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_32((int32_t volatile*)(ptr), -(int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_64((int64_t volatile*)(ptr), -(int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_sub_fetch_ptr(void *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_ptr((void* volatile*)(ptr), -(void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), -(int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), -(int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd((int32_t volatile*)(ptr), -(int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), -(int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_sub_ptr(void *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAddptr((void* volatile*)(ptr), -(void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_and_fetch_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedAnd8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedAnd16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_and_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_and_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_or_fetch_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedOr8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedOr16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_or_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_xor_fetch_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedXor8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedXor16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_xor_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 63fa60021752686ea80a591160c6b7e2f65ae2a8..61b2593bc6c3115f0d5f89c92e593082ad05fcf2 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -101,6 +101,8 @@ void osUpdate() { } } +void osCleanup() {} + bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5e859de5d602aef6077ed666575f9bb3b6ac1db8..ed93708c0775793b768f8339d33fa82bc571cb31 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -57,7 +57,7 @@ typedef int32_t SocketFd; typedef int32_t FileFd; typedef struct TdFile { - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; int refId; FileFd fd; FILE *fp; @@ -268,14 +268,14 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { autoDelFileListAdd(path); } - TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); + TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); if (pFile == NULL) { if (fd >= 0) close(fd); if (fp != NULL) fclose(fp); return NULL; } #if FILE_WITH_LOCK - pthread_rwlock_init(&(pFile->rwlock), NULL); + taosThreadRwlockInit(&(pFile->rwlock), NULL); #endif pFile->fd = fd; pFile->fp = fp; @@ -292,7 +292,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_wrlock(&((*ppFile)->rwlock)); + taosThreadRwlockWrlock(&((*ppFile)->rwlock)); #endif if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { return 0; @@ -309,10 +309,10 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { } (*ppFile)->refId = 0; #if FILE_WITH_LOCK - pthread_rwlock_unlock(&((*ppFile)->rwlock)); - pthread_rwlock_destroy(&((*ppFile)->rwlock)); + taosThreadRwlockUnlock(&((*ppFile)->rwlock)); + taosThreadRwlockDestroy(&((*ppFile)->rwlock)); #endif - free(*ppFile); + taosMemoryFree(*ppFile); *ppFile = NULL; return 0; #endif @@ -323,7 +323,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t leftbytes = count; @@ -337,13 +337,13 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { continue; } else { #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return -1; } } else if (readbytes == 0) { #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return (int64_t)(count - leftbytes); } @@ -353,7 +353,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return count; } @@ -363,12 +363,12 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t ret = pread(pFile->fd, buf, count, offset); #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return ret; } @@ -378,7 +378,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_wrlock(&(pFile->rwlock)); + taosThreadRwlockWrlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); @@ -393,7 +393,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { continue; } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return -1; } @@ -402,7 +402,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return count; } @@ -412,12 +412,12 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t ret = lseek(pFile->fd, (long)offset, whence); #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return ret; } diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 5f12f9cd3d5a2a75cd4080d1f35cdb892ea8e0b8..2df61c08d418788ecadcdff7e869a948d8c6855e 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -134,7 +134,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { char *revisedCharset = taosCharsetReplace(str); tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); - free(revisedCharset); + taosMemoryFree(revisedCharset); // printf("charset not configured, set to system default:%s", outCharset); } else { strcpy(outCharset, "UTF-8"); @@ -179,7 +179,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { char *revisedCharset = taosCharsetReplace(str); tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); - free(revisedCharset); + taosMemoryFree(revisedCharset); // printf("charset not configured, set to system default:%s", outCharset); } else { strcpy(outCharset, "UTF-8"); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c new file mode 100644 index 0000000000000000000000000000000000000000..3f47e475c36b90f59a5c4eddcefa537adaa5bf69 --- /dev/null +++ b/source/os/src/osMemory.c @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC +#include "os.h" + +#define USE_TD_MEMORY + +#define TD_MEMORY_SYMBOL ('T'<<24|'A'<<16|'O'<<8|'S') + +#define TD_MEMORY_STACK_TRACE_DEPTH 10 + +typedef struct TdMemoryInfo +{ + int32_t symbol; + void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH]; // gdb: disassemble /m 0xXXX + int32_t memorySize; +} *TdMemoryInfoPtr , TdMemoryInfo; + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +#else + +#include +#define STACKCALL __attribute__((regparm(1), noinline)) +void **STACKCALL taosGetEbp(void) { + void **ebp = NULL; + __asm__ __volatile__("mov %%rbp, %0;\n\t" + : "=m"(ebp) /* output */ + : /* input */ + : "memory"); /* not affect register */ + return (void **)(*ebp); +} +int32_t taosBackTrace(void **buffer, int32_t size) { + int32_t frame = 0; + void **ebp; + void **ret = NULL; + unsigned long long func_frame_distance = 0; + if (buffer != NULL && size > 0) { + ebp = taosGetEbp(); + func_frame_distance = (unsigned long long)(*ebp) - (unsigned long long)ebp; + while (ebp && frame < size && (func_frame_distance < (1ULL << 24)) // assume function ebp more than 16M + && (func_frame_distance > 0)) { + ret = ebp + 1; + buffer[frame++] = *ret; + ebp = (void **)(*ebp); + func_frame_distance = (unsigned long long)(*ebp) - (unsigned long long)ebp; + } + } + return frame; +} +#endif + +// char **taosBackTraceSymbols(int32_t *size) { +// void *buffer[20] = {NULL}; +// *size = taosBackTrace(buffer, 20); +// return backtrace_symbols(buffer, *size); +// } + +void *taosMemoryMalloc(int32_t size) { + void *tmp = malloc(size + sizeof(TdMemoryInfo)); + if (tmp == NULL) return NULL; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)tmp; + pTdMemoryInfo->memorySize = size; + pTdMemoryInfo->symbol = TD_MEMORY_SYMBOL; + taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void *taosMemoryCalloc(int32_t num, int32_t size) { + int32_t memorySize = num * size; + char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); + if (tmp == NULL) return NULL; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)tmp; + pTdMemoryInfo->memorySize = memorySize; + pTdMemoryInfo->symbol = TD_MEMORY_SYMBOL; + taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void *taosMemoryRealloc(void *ptr, int32_t size) { + if (ptr == NULL) return taosMemoryMalloc(size); + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + + TdMemoryInfo tdMemoryInfo; + memcpy(&tdMemoryInfo, pTdMemoryInfo, sizeof(TdMemoryInfo)); + + void *tmp = realloc(pTdMemoryInfo, size + sizeof(TdMemoryInfo)); + if (tmp == NULL) return NULL; + + memcpy(tmp, &tdMemoryInfo, sizeof(TdMemoryInfo)); + ((TdMemoryInfoPtr)tmp)->memorySize = size; + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void taosMemoryFree(const void *ptr) { + if (ptr == NULL) return; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + if(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL) { + memset(pTdMemoryInfo, 0, sizeof(TdMemoryInfo)); + free(pTdMemoryInfo); + } else { + free((void*)ptr); + } +} + +int32_t taosMemorySize(void *ptr) { + if (ptr == NULL) return 0; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + + return pTdMemoryInfo->memorySize; +} \ No newline at end of file diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index e5b506e8d835375205a2ffc18a7242d1e65506a1..1297fdbc276cc3a9a4611fa296ba495b452287e3 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -24,11 +24,11 @@ #include -bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } +bool taosCheckPthreadValid(TdThread thread) { return thread.p != NULL; } -void taosResetPthread(pthread_t* thread) { thread->p = 0; } +void taosResetPthread(TdThread* thread) { thread->p = 0; } -int64_t taosGetPthreadId(pthread_t thread) { +int64_t taosGetPthreadId(TdThread thread) { #ifdef PTW32_VERSION return pthread_getw32threadid_np(thread); #else @@ -38,7 +38,7 @@ int64_t taosGetPthreadId(pthread_t thread) { int64_t taosGetSelfPthreadId() { return GetCurrentThreadId(); } -bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } +bool taosComparePthread(TdThread first, TdThread second) { return first.p == second.p; } int32_t taosGetPId() { return GetCurrentProcessId(); } @@ -84,10 +84,9 @@ int32_t tsem_wait(tsem_t* sem) { #include #include #include -#include -static pthread_t sem_thread; -static pthread_once_t sem_once; +static TdThread sem_thread; +static TdThreadOnce sem_once; static task_t sem_port; static volatile int sem_inited = 0; static semaphore_t sem_exit; @@ -110,7 +109,7 @@ static void *sem_thread_routine(void *arg) { static void once_init(void) { int r = 0; - r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); + r = taosThreadCreate(&sem_thread, NULL, sem_thread_routine, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); return; @@ -123,8 +122,8 @@ static void once_init(void) { struct tsem_s { #ifdef SEM_USE_PTHREAD - pthread_mutex_t lock; - pthread_cond_t cond; + TdThreadMutex lock; + TdThreadCond cond; volatile int64_t val; #elif defined(SEM_USE_POSIX) size_t id; @@ -144,19 +143,19 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - struct tsem_s *p = (struct tsem_s *)calloc(1, sizeof(*p)); + struct tsem_s *p = (struct tsem_s *)taosMemoryCalloc(1, sizeof(*p)); if (!p) { fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } #ifdef SEM_USE_PTHREAD - int r = pthread_mutex_init(&p->lock, NULL); + int r = taosThreadMutexInit(&p->lock, NULL); do { if (r) break; - r = pthread_cond_init(&p->cond, NULL); + r = taosThreadCondInit(&p->cond, NULL); if (r) { - pthread_mutex_destroy(&p->lock); + taosThreadMutexDestroy(&p->lock); break; } p->val = value; @@ -186,7 +185,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { abort(); } while (p->sem == SEM_FAILED); #elif defined(SEM_USE_SEM) - pthread_once(&sem_once, once_init); + taosThreadOnce(&sem_once, once_init); if (sem_inited != 1) { fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); errno = ENOMEM; @@ -224,18 +223,18 @@ int tsem_wait(tsem_t *sem) { abort(); } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val -= 1; if (p->val < 0) { - if (pthread_cond_wait(&p->cond, &p->lock)) { + if (taosThreadCondWait(&p->cond, &p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -260,18 +259,18 @@ int tsem_post(tsem_t *sem) { abort(); } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val += 1; if (p->val <= 0) { - if (pthread_cond_signal(&p->cond)) { + if (taosThreadCondSignal(&p->cond)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -299,20 +298,20 @@ int tsem_destroy(tsem_t *sem) { return 0; } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->valid = 0; - if (pthread_cond_destroy(&p->cond)) { + if (taosThreadCondDestroy(&p->cond)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - if (pthread_mutex_destroy(&p->lock)) { + if (taosThreadMutexDestroy(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -332,29 +331,29 @@ int tsem_destroy(tsem_t *sem) { #endif // SEM_USE_PTHREAD p->valid = 0; - free(p); + taosMemoryFree(p); *sem = NULL; return 0; } -bool taosCheckPthreadValid(pthread_t thread) { +bool taosCheckPthreadValid(TdThread thread) { uint64_t id = 0; - int r = pthread_threadid_np(thread, &id); + int r = TdThreadhreadid_np(thread, &id); return r ? false : true; } int64_t taosGetSelfPthreadId() { uint64_t id; - pthread_threadid_np(0, &id); + TdThreadhreadid_np(0, &id); return (int64_t)id; } -int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } +int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } -void taosResetPthread(pthread_t *thread) { *thread = NULL; } +void taosResetPthread(TdThread *thread) { *thread = NULL; } -bool taosComparePthread(pthread_t first, pthread_t second) { return pthread_equal(first, second) ? true : false; } +bool taosComparePthread(TdThread first, TdThread second) { return taosThreadEqual(first, second) ? true : false; } int32_t taosGetPId() { return (int32_t)getpid(); } @@ -378,7 +377,7 @@ int32_t taosGetAppName(char *name, int32_t *len) { #include #include -bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } +bool taosCheckPthreadValid(TdThread thread) { return thread != 0; } int64_t taosGetSelfPthreadId() { static __thread int id = 0; @@ -387,9 +386,9 @@ int64_t taosGetSelfPthreadId() { return id; } -int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } -void taosResetPthread(pthread_t* thread) { *thread = 0; } -bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } +int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } +void taosResetPthread(TdThread* thread) { *thread = 0; } +bool taosComparePthread(TdThread first, TdThread second) { return first == second; } int32_t taosGetPId() { return getpid(); } int32_t taosGetAppName(char* name, int32_t* len) { diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index e3b1015d889f9a9a16a99eda1354ed6b48939e49..b130689d4b5aa7bba55d341796cf6416594209d5 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -47,28 +47,17 @@ #endif #endif -typedef int32_t SocketFd; -typedef SocketFd EpollFd; - typedef struct TdSocketServer { #if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif int refId; SocketFd fd; } *TdSocketServerPtr, TdSocketServer; -typedef struct TdSocket { -#if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; -#endif - int refId; - SocketFd fd; -} *TdSocketPtr, TdSocket; - typedef struct TdEpoll { #if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif int refId; EpollFd fd; @@ -128,7 +117,7 @@ int32_t taosCloseSocket(TdSocketPtr *ppSocket) { } code = taosCloseSocketNoCheck1((*ppSocket)->fd); (*ppSocket)->fd = -1; - free(*ppSocket); + taosMemoryFree(*ppSocket); return code; } int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { @@ -138,7 +127,7 @@ int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { } code = taosCloseSocketNoCheck1((*ppSocketServer)->fd); (*ppSocketServer)->fd = -1; - free(*ppSocketServer); + taosMemoryFree(*ppSocketServer); return code; } @@ -216,12 +205,6 @@ int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { #endif } -#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) - #if defined(_TD_GO_DLL_) - uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } - #endif -#endif - void taosWinSocketInit1() { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) static char flag = 0; @@ -457,7 +440,7 @@ TdSocketPtr taosOpenUdpSocket(uint32_t ip, uint16_t port) { return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -501,7 +484,7 @@ TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -671,7 +654,7 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -720,7 +703,7 @@ TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct s return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -775,7 +758,7 @@ void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = taosThreadSigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { // printf("failed to block SIGPIPE"); } @@ -893,7 +876,7 @@ void taosSetMaskSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); + int32_t rc = taosThreadSigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { // printf("failed to setmask SIGPIPE"); } @@ -918,7 +901,7 @@ TdEpollPtr taosCreateEpoll(int32_t size) { return NULL; } - TdEpollPtr pEpoll = (TdEpollPtr)malloc(sizeof(TdEpoll)); + TdEpollPtr pEpoll = (TdEpollPtr)taosMemoryMalloc(sizeof(TdEpoll)); if (pEpoll == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -956,6 +939,6 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { } code = taosCloseSocketNoCheck1((*ppEpoll)->fd); (*ppEpoll)->fd = -1; - free(*ppEpoll); + taosMemoryFree(*ppEpoll); return code; } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index d3d1ab5ddaf0c97ce6d2d8e0f5037509721e5a00..5adb564e52f0d050fe7885f21d8dd11349518763 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -15,7 +15,6 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE -#include #include "os.h" #ifndef DISALLOW_NCHAR_WITHOUT_ICONV @@ -50,8 +49,8 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { #if 0 int32_t ucs4_max_len = bytes + 4; - char *f1_mbs = calloc(bytes, 1); - char *f2_mbs = calloc(bytes, 1); + char *f1_mbs = taosMemoryCalloc(bytes, 1); + char *f2_mbs = taosMemoryCalloc(bytes, 1); if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { return -1; } @@ -59,15 +58,15 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { return -1; } int32_t ret = strcmp(f1_mbs, f2_mbs); - free(f1_mbs); - free(f2_mbs); + taosMemoryFree(f1_mbs); + taosMemoryFree(f2_mbs); return ret; #endif } TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { - assert(malloc_usable_size(target_ucs4)>=len_ucs4*sizeof(TdUcs4)); + assert(taosMemorySize(target_ucs4)>=len_ucs4*sizeof(TdUcs4)); return memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4)); } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 9bff7605097f20794b9a2d7b89e1127dc3a7d3b7..eb5712d2e541d2827f115b1e3bddc7456608934b 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -145,7 +145,7 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; } @@ -174,7 +174,7 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; } @@ -268,7 +268,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; #else @@ -293,7 +293,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; #endif @@ -324,7 +324,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; @@ -351,7 +351,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; @@ -486,7 +486,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { char tmp[10]; sscanf(line, "%s %" PRId64, tmp, usedKB); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; #endif @@ -606,7 +606,7 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int if (readIndex >= 4) break; } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); if (readIndex < 4) { @@ -665,7 +665,7 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { *transmit_bytes = o_tbytes; } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 8ccbe9d78004760ab9f2377ede3a41dbfbe87f46..a36b3d41d032387d7f6730371d62eabd8052ebfe 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -29,61 +29,6 @@ struct termios oldtio; #endif -int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize) { -#if defined(WINDOWS) - FILE *fp; - if (cmd == NULL) { - // printf("taosSystem cmd is NULL!"); - return -1; - } - - if ((fp = _popen(cmd, "r")) == NULL) { - // printf("popen cmd:%s error: %s", cmd, strerror(errno)); - return -1; - } else { - while (fgets(buf, bufSize, fp)) { - // printf("popen result:%s", buf); - } - - if (!_pclose(fp)) { - // printf("close popen file pointer fp error!"); - return -1; - } else { - // printf("popen res is :%d", res); - } - - return 0; - } -#elif defined(_TD_DARWIN_64) - printf("no support funtion"); - return -1; -#else - FILE *fp; - int32_t res; - if (cmd == NULL) { - // printf("taosSystem cmd is NULL!"); - return -1; - } - - if ((fp = popen(cmd, "r")) == NULL) { - // printf("popen cmd:%s error: %s", cmd, strerror(errno)); - return -1; - } else { - while (fgets(buf, bufSize, fp)) { - // printf("popen result:%s", buf); - } - - if ((res = pclose(fp)) == -1) { - // printf("close popen file pointer fp error!"); - } else { - // printf("popen res is :%d", res); - } - - return res; - } -#endif -} - void* taosLoadDll(const char* filename) { #if defined(WINDOWS) return NULL; @@ -103,7 +48,7 @@ void* taosLoadDll(const char* filename) { } void* taosLoadSym(void* handle, char* name) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if defined(WINDOWS) return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -123,7 +68,7 @@ void* taosLoadSym(void* handle, char* name) { } void taosCloseDll(void* handle) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if defined(WINDOWS) return; #elif defined(_TD_DARWIN_64) return; @@ -135,7 +80,7 @@ void taosCloseDll(void* handle) { } int taosSetConsoleEcho(bool on) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if defined(WINDOWS) HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); DWORD mode = 0; GetConsoleMode(hStdin, &mode); @@ -146,28 +91,6 @@ int taosSetConsoleEcho(bool on) { } SetConsoleMode(hStdin, mode); - return 0; -#elif defined(_TD_DARWIN_64) -#define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) - int err; - struct termios term; - - if (tcgetattr(STDIN_FILENO, &term) == -1) { - perror("Cannot get the attribution of the terminal"); - return -1; - } - - if (on) - term.c_lflag |= ECHOFLAGS; - else - term.c_lflag &= ~ECHOFLAGS; - - err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term); - if (err == -1 && err == EINTR) { - perror("Cannot set the attribution of the terminal"); - return -1; - } - return 0; #else #define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) @@ -186,7 +109,7 @@ int taosSetConsoleEcho(bool on) { err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term); if (err == -1 || err == EINTR) { - //printf("Cannot set the attribution of the terminal"); + printf("Cannot set the attribution of the terminal"); return -1; } @@ -195,35 +118,8 @@ int taosSetConsoleEcho(bool on) { } void setTerminalMode() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -#elif defined(_TD_DARWIN_64) - struct termios newtio; - - /* if (atexit() != 0) { */ - /* fprintf(stderr, "Error register exit function!\n"); */ - /* exit(EXIT_FAILURE); */ - /* } */ - - memcpy(&newtio, &oldtio, sizeof(oldtio)); - - // Set new terminal attributes. - newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP); - newtio.c_iflag |= IGNBRK; - - // newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET); - newtio.c_oflag |= OPOST; - newtio.c_oflag |= ONLCR; - newtio.c_oflag &= ~(OCRNL | ONLRET); - - newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG); - newtio.c_cc[VMIN] = 1; - newtio.c_cc[VTIME] = 0; +#if defined(WINDOWS) - if (tcsetattr(0, TCSANOW, &newtio) != 0) { - fprintf(stderr, "Fail to set terminal properties!\n"); - exit(EXIT_FAILURE); - } #else struct termios newtio; @@ -256,19 +152,7 @@ void setTerminalMode() { int32_t getOldTerminalMode() { #if defined(WINDOWS) - -#elif defined(_TD_DARWIN_64) - /* Make sure stdin is a terminal. */ - if (!isatty(STDIN_FILENO)) { - return -1; - } - - // Get the parameter of current terminal - if (tcgetattr(0, &oldtio) != 0) { - return -1; - } - - return 1; + #else /* Make sure stdin is a terminal. */ if (!isatty(STDIN_FILENO)) { @@ -285,13 +169,8 @@ int32_t getOldTerminalMode() { } void resetTerminalMode() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if defined(WINDOWS) -#elif defined(_TD_DARWIN_64) - if (tcsetattr(0, TCSANOW, &oldtio) != 0) { - fprintf(stderr, "Fail to reset the terminal properties!\n"); - exit(EXIT_FAILURE); - } #else if (tcsetattr(0, TCSANOW, &oldtio) != 0) { fprintf(stderr, "Fail to reset the terminal properties!\n"); diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c new file mode 100644 index 0000000000000000000000000000000000000000..716080274ba74315d6aef9a7bf40ee70738648aa --- /dev/null +++ b/source/os/src/osThread.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC +#include "os.h" + +// int32_t taosThreadSetnameNp(TdThread thread, const char *name) { +// return pthread_setname_np(thread,name); +// } + +int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared) { + return pthread_spin_init(lock, pshared); +} + +int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) { + return pthread_mutex_init(mutex, attr); +} + +int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { + return pthread_spin_destroy(lock); +} + +int32_t taosThreadMutexDestroy(TdThreadMutex * mutex) { + return pthread_mutex_destroy(mutex); +} + +int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { + return pthread_spin_lock(lock); +} + +int32_t taosThreadMutexLock(TdThreadMutex *mutex) { + return pthread_mutex_lock(mutex); +} + +int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { + return pthread_spin_unlock(lock); +} + +int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { + return pthread_mutex_unlock(mutex); +} + +int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_rdlock(rwlock); +} + +int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_wrlock(rwlock); +} + +int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_unlock(rwlock); +} + +void taosThreadTestCancel(void) { + return pthread_testcancel(); +} + +int32_t taosThreadAttrInit(TdThreadAttr *attr) { + return pthread_attr_init(attr); +} + +int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg) { + return pthread_create(tid, attr, start, arg); +} + +int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void)) { + return pthread_once(onceControl, initRoutine); +} + +int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState) { + return pthread_attr_setdetachstate(attr, detachState); +} + +int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { + return pthread_attr_destroy(attr); +} + +int32_t taosThreadJoin(TdThread thread, void **pValue) { + return pthread_join(thread, pValue); +} + +int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr) { + return pthread_rwlock_init(rwlock, attr); +} + +int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) { + return pthread_rwlock_destroy(rwlock); +} + +int32_t taosThreadCondSignal(TdThreadCond *cond) { + return pthread_cond_signal(cond); +} + +int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) { + return pthread_cond_init(cond, attr); +} + +int32_t taosThreadCondBroadcast(TdThreadCond *cond) { + return pthread_cond_broadcast(cond); +} + +int32_t taosThreadCondDestroy(TdThreadCond *cond) { + return pthread_cond_destroy(cond); +} + +int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { + return pthread_cond_wait(cond, mutex); +} + +TdThread taosThreadSelf(void) { + return pthread_self(); +} + +// int32_t taosThreadGetW32ThreadIdNp(TdThread thread) { +// return pthread_getw32threadid_np(thread); +// } + +int32_t taosThreadEqual(TdThread t1, TdThread t2) { + return pthread_equal(t1, t2); +} + +int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset) { + return pthread_sigmask(how, set, oset); +} + +int32_t taosThreadCancel(TdThread thread) { + return pthread_cancel(thread); +} + +int32_t taosThreadKill(TdThread thread, int sig) { + return pthread_kill(thread, sig); +} \ No newline at end of file diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index c95ca72bd57fd9e8165cda69edcdb003992816c0..e6e3ec7962192a3aca4da49f9bfe9394d7bdc73b 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -44,7 +44,7 @@ static MMRESULT timerId; static void (*timer_callback)(int); static int timer_ms = 0; -static pthread_t timer_thread; +static TdThread timer_thread; static int timer_kq = -1; static volatile int timer_stop = 0; @@ -83,7 +83,7 @@ static void taosDeleteTimer(void *tharg) { timer_delete(*pTimer); } -static pthread_t timerThread; +static TdThread timerThread; static timer_t timerId; static volatile bool stopTimer = false; static void * taosProcessAlarmSignal(void *tharg) { @@ -112,7 +112,7 @@ static void * taosProcessAlarmSignal(void *tharg) { // printf("Failed to create timer"); } - pthread_cleanup_push(taosDeleteTimer, &timerId); + taosThreadCleanupPush(taosDeleteTimer, &timerId); struct itimerspec ts; ts.it_value.tv_sec = 0; @@ -136,7 +136,7 @@ static void * taosProcessAlarmSignal(void *tharg) { callback(0); } - pthread_cleanup_pop(1); + taosThreadCleanupPop(1); return NULL; } @@ -165,7 +165,7 @@ int taosInitTimer(void (*callback)(int), int ms) { abort(); } - r = pthread_create(&timer_thread, NULL, timer_routine, NULL); + r = taosThreadCreate(&timer_thread, NULL, timer_routine, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment @@ -174,10 +174,10 @@ int taosInitTimer(void (*callback)(int), int ms) { return 0; #else stopTimer = false; - pthread_attr_t tattr; - pthread_attr_init(&tattr); - int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback); - pthread_attr_destroy(&tattr); + TdThreadAttr tattr; + taosThreadAttrInit(&tattr); + int code = taosThreadCreate(&timerThread, &tattr, taosProcessAlarmSignal, callback); + taosThreadAttrDestroy(&tattr); if (code != 0) { // printf("failed to create timer thread"); return -1; @@ -195,7 +195,7 @@ void taosUninitTimer() { #elif defined(_TD_DARWIN_64) int r = 0; timer_stop = 1; - r = pthread_join(timer_thread, NULL); + r = taosThreadJoin(timer_thread, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment @@ -207,7 +207,7 @@ void taosUninitTimer() { stopTimer = true; // printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); - pthread_join(timerThread, NULL); + taosThreadJoin(timerThread, NULL); #endif } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index dd34d77b2347790bcd4516cd9cc7edfdc2e603af..bdafa63d643b0a44be0bd1bfdbc7ed53540a584b 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 08fa0a240903736007605d349f11b953c4aacc64..7a47639e75f2e18e6ed47a19066c0a69ea336503 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -5,10 +5,6 @@ target_include_directories( util PUBLIC "${CMAKE_SOURCE_DIR}/include/util" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -IF(${TD_WINDOWS}) - PRIVATE "${CMAKE_SOURCE_DIR}/contrib/pthread-win32" - PRIVATE "${CMAKE_SOURCE_DIR}/contrib/gnuregex" -ENDIF () ) target_link_libraries( util diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index cb0a2d25a90a8999399e2e3bbfc3b4a4c838b0bd..48a0f327c4111d89dfd99437fc7c763d2d65b101 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -153,9 +153,9 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, cons } void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { - char *buf = calloc(1, size); // prepare the swap buffer + char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); - tfree(buf); + taosMemoryFreeClear(buf); } void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t compar, int32_t flags) { @@ -239,7 +239,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const child = 2 * parent + 1; if (swap == NULL) { - buf = calloc(1, size); + buf = taosMemoryCalloc(1, size); if (buf == NULL) { return; } @@ -288,7 +288,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const } if (swap == NULL) { - tfree(buf); + taosMemoryFreeClear(buf); } } } @@ -304,13 +304,13 @@ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, } /* - char *buf = calloc(1, size); + char *buf = taosMemoryCalloc(1, size); for (i = len - 1; i > 0; i--) { doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); } - tfree(buf); + taosMemoryFreeClear(buf); */ } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 1127a212559ab832a6afdb367c8ab5d4b46a6877..a74b26a38666afe4e54817e9f8a773b156476e8b 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -23,15 +23,15 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { size = TARRAY_MIN_SIZE; } - SArray* pArray = malloc(sizeof(SArray)); + SArray* pArray = taosMemoryMalloc(sizeof(SArray)); if (pArray == NULL) { return NULL; } pArray->size = 0; - pArray->pData = calloc(size, elemSize); + pArray->pData = taosMemoryCalloc(size, elemSize); if (pArray->pData == NULL) { - free(pArray); + taosMemoryFree(pArray); return NULL; } @@ -46,7 +46,7 @@ static int32_t taosArrayResize(SArray* pArray) { size_t size = pArray->capacity; size = (size << 1u); - void* tmp = realloc(pArray->pData, size * pArray->elemSize); + void* tmp = taosMemoryRealloc(pArray->pData, size * pArray->elemSize); if (tmp == NULL) { // reallocate failed, the original buffer remains return -1; } @@ -64,7 +64,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { tsize = (tsize << 1u); } - pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize); + pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); if (pArray->pData == NULL) { return -1; } @@ -305,8 +305,8 @@ void taosArrayClear(SArray* pArray) { void* taosArrayDestroy(SArray* pArray) { if (pArray) { - free(pArray->pData); - free(pArray); + taosMemoryFree(pArray->pData); + taosMemoryFree(pArray); } return NULL; diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index c5119e8b2d43cce3e0cc0781c54206da8f9f0bea..a2f4ddbc51599ebc3909440b628d58819515fb2f 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -20,7 +20,7 @@ static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01 char *base64_encode(const uint8_t *value, int32_t vlen) { uint8_t oval = 0; - char *result = (char *)malloc((size_t)(vlen * 4) / 3 + 10); + char *result = (char *)taosMemoryMalloc((size_t)(vlen * 4) / 3 + 10); char *out = result; while (vlen >= 3) { *out++ = basis_64[value[0] >> 2]; @@ -53,7 +53,7 @@ static signed char index_64[128] = { uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { int32_t c1, c2, c3, c4; - uint8_t *result = (uint8_t *)malloc((size_t)(inlen * 3) / 4 + 1); + uint8_t *result = (uint8_t *)taosMemoryMalloc((size_t)(inlen * 3) / 4 + 1); uint8_t *out = result; *outlen = 0; @@ -93,7 +93,7 @@ uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { } base64_decode_error: - free(result); + taosMemoryFree(result); result = 0; *outlen = 0; diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index dc8c4b70c5107358a742e16ceef67584a111ff69..8552ccac2c66600ab851fb9770887f6e8a431c78 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -213,7 +213,7 @@ double tbufReadDouble(SBufferReader* buf) { // writer functions void tbufCloseWriter(SBufferWriter* buf) { - tfree(buf->data); + taosMemoryFreeClear(buf->data); // (*buf->allocator)( buf->data, 0 ); // potential memory leak. buf->data = NULL; buf->pos = 0; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 51ea2b1f13c6e8fab1a9ecca59dd6d44e51d9c6d..c4e85a425e1c9af3beb8146ca58685756989d067 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -22,9 +22,9 @@ #define CACHE_MAX_CAPACITY 1024*1024*16 #define CACHE_DEFAULT_CAPACITY 1024*4 -static pthread_t cacheRefreshWorker = {0}; -static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; +static TdThread cacheRefreshWorker = {0}; +static TdThreadOnce cacheThreadInit = PTHREAD_ONCE_INIT; +static TdThreadMutex guard = PTHREAD_MUTEX_INITIALIZER; static SArray *pCacheArrayList = NULL; static bool stopRefreshWorker = false; static bool refreshWorkerNormalStopped = false; @@ -90,13 +90,13 @@ struct SCacheObj { STrashElem *pTrash; uint8_t deleting; // set the deleting flag to stop refreshing ASAP. - pthread_t refreshWorker; + TdThread refreshWorker; bool extendLifespan; // auto extend life span when one item is accessed. int64_t checkTick; // tick used to record the check times of the refresh threads #if defined(LINUX) - pthread_rwlock_t lock; + TdThreadRwlock lock; #else - pthread_mutex_t lock; + TdThreadMutex lock; #endif }; @@ -109,33 +109,33 @@ typedef struct SCacheObjTravSup { static FORCE_INLINE void __trashcan_wr_lock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_wrlock(&pCacheObj->lock); + taosThreadRwlockWrlock(&pCacheObj->lock); #else - pthread_mutex_lock(&pCacheObj->lock); + taosThreadMutexLock(&pCacheObj->lock); #endif } static FORCE_INLINE void __trashcan_unlock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_unlock(&pCacheObj->lock); + taosThreadRwlockUnlock(&pCacheObj->lock); #else - pthread_mutex_unlock(&pCacheObj->lock); + taosThreadMutexUnlock(&pCacheObj->lock); #endif } static FORCE_INLINE int32_t __trashcan_lock_init(SCacheObj *pCacheObj) { #if defined(LINUX) - return pthread_rwlock_init(&pCacheObj->lock, NULL); + return taosThreadRwlockInit(&pCacheObj->lock, NULL); #else - return pthread_mutex_init(&pCacheObj->lock, NULL); + return taosThreadMutexInit(&pCacheObj->lock, NULL); #endif } static FORCE_INLINE void __trashcan_lock_destroy(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_destroy(&pCacheObj->lock); + taosThreadRwlockDestroy(&pCacheObj->lock); #else - pthread_mutex_destroy(&pCacheObj->lock); + taosThreadMutexDestroy(&pCacheObj->lock); #endif } @@ -154,20 +154,20 @@ static void *taosCacheTimedRefresh(void *handle); static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&cacheRefreshWorker, &thattr, taosCacheTimedRefresh, NULL); - pthread_attr_destroy(&thattr); + taosThreadCreate(&cacheRefreshWorker, &thattr, taosCacheTimedRefresh, NULL); + taosThreadAttrDestroy(&thattr); } -pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) { - pthread_once(&cacheThreadInit, doInitRefreshThread); +TdThread doRegisterCacheObj(SCacheObj *pCacheObj) { + taosThreadOnce(&cacheThreadInit, doInitRefreshThread); - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); taosArrayPush(pCacheArrayList, &pCacheObj); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); return cacheRefreshWorker; } @@ -221,7 +221,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode * pCacheObj->freeFp(pNode->data); } - free(pNode); + taosMemoryFree(pNode); } static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) { @@ -255,8 +255,8 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem pCacheObj->freeFp(pElem->pData->data); } - free(pElem->pData); - free(pElem); + taosMemoryFree(pElem->pData); + taosMemoryFree(pElem); } static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { @@ -358,7 +358,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - SCacheObj *pCacheObj = (SCacheObj *)calloc(1, sizeof(SCacheObj)); + SCacheObj *pCacheObj = (SCacheObj *)taosMemoryCalloc(1, sizeof(SCacheObj)); if (pCacheObj == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; @@ -366,9 +366,9 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext // TODO add the auto extend procedure pCacheObj->capacity = 4096; - pCacheObj->pEntryList = calloc(pCacheObj->capacity, sizeof(SCacheEntry)); + pCacheObj->pEntryList = taosMemoryCalloc(pCacheObj->capacity, sizeof(SCacheEntry)); if (pCacheObj->pEntryList == NULL) { - free(pCacheObj); + taosMemoryFree(pCacheObj); uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } @@ -381,8 +381,8 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access if (__trashcan_lock_init(pCacheObj) != 0) { - tfree(pCacheObj->pEntryList); - free(pCacheObj); + taosMemoryFreeClear(pCacheObj->pEntryList); + taosMemoryFree(pCacheObj); uError("failed to init lock, reason:%s", strerror(errno)); return NULL; @@ -432,7 +432,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v } atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); - tfree(pNode); + taosMemoryFreeClear(pNode); } else { taosAddToTrashcan(pCacheObj, pNode); uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pNode->data); @@ -456,7 +456,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen } if (pCacheObj->numOfElems == 0) { - atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); + atomic_add_fetch_64(&pCacheObj->statistics.missCount, 1); return NULL; } @@ -475,15 +475,15 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen void *pData = (pNode != NULL) ? pNode->data : NULL; if (pData != NULL) { - atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); + atomic_add_fetch_64(&pCacheObj->statistics.hitCount, 1); uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, T_REF_VAL_GET(pNode)); } else { - atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); + atomic_add_fetch_64(&pCacheObj->statistics.missCount, 1); uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); } - atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); + atomic_add_fetch_64(&pCacheObj->statistics.totalAccess, 1); return pData; } @@ -625,7 +625,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { pCacheObj->freeFp(pNode->data); } - free(pNode); + taosMemoryFree(pNode); } } @@ -703,7 +703,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { size_t sizeInBytes = size + sizeof(SCacheNode) + keyLen; - SCacheNode *pNewNode = calloc(1, sizeInBytes); + SCacheNode *pNewNode = taosMemoryCalloc(1, sizeInBytes); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; uError("failed to allocate memory, reason:%s", strerror(errno)); @@ -735,7 +735,7 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) { } __trashcan_wr_lock(pCacheObj); - STrashElem *pElem = calloc(1, sizeof(STrashElem)); + STrashElem *pElem = taosMemoryCalloc(1, sizeof(STrashElem)); pElem->pData = pNode; pElem->prev = NULL; pElem->next = NULL; @@ -802,9 +802,9 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { __trashcan_lock_destroy(pCacheObj); - tfree(pCacheObj->pEntryList); - tfree(pCacheObj->name); - free(pCacheObj); + taosMemoryFreeClear(pCacheObj->pEntryList); + taosMemoryFreeClear(pCacheObj->name); + taosMemoryFree(pCacheObj); } static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { @@ -836,19 +836,19 @@ void *taosCacheTimedRefresh(void *handle) { goto _end; } - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); size_t size = taosArrayGetSize(pCacheArrayList); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); count += 1; for (int32_t i = 0; i < size; ++i) { - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); SCacheObj *pCacheObj = taosArrayGetP(pCacheArrayList, i); if (pCacheObj == NULL) { uError("object is destroyed. ignore and try next"); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); continue; } @@ -860,11 +860,11 @@ void *taosCacheTimedRefresh(void *handle) { uDebug("%s is destroying, remove it from refresh list, remain cache obj:%" PRIzu, pCacheObj->name, size); pCacheObj->deleting = 0; // reset the deleting flag to enable pCacheObj to continue releasing resources. - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); continue; } - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); if ((count % pCacheObj->checkTick) != 0) { continue; @@ -892,7 +892,7 @@ _end: taosArrayDestroy(pCacheArrayList); pCacheArrayList = NULL; - pthread_mutex_destroy(&guard); + taosThreadMutexDestroy(&guard); refreshWorkerNormalStopped = true; uDebug("cache refresh thread quits"); @@ -918,7 +918,7 @@ size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) { SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj) { ASSERT(pCacheObj != NULL); - SCacheIter* pIter = calloc(1, sizeof(SCacheIter)); + SCacheIter* pIter = taosMemoryCalloc(1, sizeof(SCacheIter)); pIter->pCacheObj = (SCacheObj*) pCacheObj; pIter->entryIndex = -1; pIter->index = -1; @@ -955,7 +955,7 @@ bool taosCacheIterNext(SCacheIter* pIter) { } if (pIter->numOfObj < pEntry->num) { - char *tmp = realloc(pIter->pCurrent, pEntry->num * POINTER_BYTES); + char *tmp = taosMemoryRealloc(pIter->pCurrent, pEntry->num * POINTER_BYTES); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosRUnLockLatch(&pEntry->latch); @@ -1001,6 +1001,6 @@ void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* len) { } void taosCacheDestroyIter(SCacheIter* pIter) { - tfree(pIter->pCurrent); - tfree(pIter); + taosMemoryFreeClear(pIter->pCurrent); + taosMemoryFreeClear(pIter); } \ No newline at end of file diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index ff7d2cf733c6af1043957d83d0dc24a8073db571..c98f6eb9be326317dd35dd678fed724ec30546ca 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -355,12 +355,12 @@ int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight) { int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { size_t sz = varDataLen(pRight); - char *pattern = malloc(sz + 1); + char *pattern = taosMemoryMalloc(sz + 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); pattern[sz] = 0; sz = varDataLen(pLeft); - char *str = malloc(sz + 1); + char *str = taosMemoryMalloc(sz + 1); memcpy(str, varDataVal(pLeft), sz); str[sz] = 0; @@ -373,8 +373,8 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf); regfree(®ex); - free(str); - free(pattern); + taosMemoryFree(str); + taosMemoryFree(pattern); return 1; } @@ -385,8 +385,8 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { } int32_t result = (errCode == 0) ? 0 : 1; regfree(®ex); - free(str); - free(pattern); + taosMemoryFree(str); + taosMemoryFree(pattern); return result; } @@ -401,17 +401,17 @@ int32_t compareStrPatternMatch(const void *pLeft, const void *pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); - char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); + char *pattern = taosMemoryCalloc(varDataLen(pRight) + 1, sizeof(char)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); + char *buf = taosMemoryMalloc(sz + 1); memcpy(buf, varDataVal(pLeft), sz); buf[sz] = 0; int32_t ret = patternMatch(pattern, buf, sz, &pInfo); - free(buf); - free(pattern); + taosMemoryFree(buf); + taosMemoryFree(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } @@ -424,11 +424,11 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - char *pattern = calloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); + char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); - free(pattern); + taosMemoryFree(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 404c9c8f7184b937fa28add3794868397a257085..96bb638b9555f0612032670b59e0f03a9444e2a8 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -29,7 +29,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); SConfig *cfgInit() { - SConfig *pCfg = calloc(1, sizeof(SConfig)); + SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig)); if (pCfg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -37,7 +37,7 @@ SConfig *cfgInit() { pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); if (pCfg->array == NULL) { - free(pCfg); + taosMemoryFree(pCfg); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -75,7 +75,7 @@ int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) { static void cfgFreeItem(SConfigItem *pItem) { if (pItem->dtype == CFG_DTYPE_STRING || pItem->dtype == CFG_DTYPE_DIR || pItem->dtype == CFG_DTYPE_LOCALE || pItem->dtype == CFG_DTYPE_CHARSET || pItem->dtype == CFG_DTYPE_TIMEZONE) { - tfree(pItem->str); + taosMemoryFreeClear(pItem->str); } if (pItem->array) { taosArrayDestroy(pItem->array); @@ -89,10 +89,10 @@ void cfgCleanup(SConfig *pCfg) { for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); cfgFreeItem(pItem); - tfree(pItem->name); + taosMemoryFreeClear(pItem->name); } taosArrayDestroy(pCfg->array); - free(pCfg); + taosMemoryFree(pCfg); } } @@ -145,7 +145,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } - tfree(pItem->str); + taosMemoryFreeClear(pItem->str); pItem->str = strdup(fullDir); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -224,7 +224,7 @@ static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType s return -1; } - free(pItem->str); + taosMemoryFree(pItem->str); pItem->str = tmp; pItem->stype = stype; return 0; @@ -366,9 +366,9 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { if (taosArrayPush(pCfg->array, pItem) == NULL) { if (pItem->dtype == CFG_DTYPE_STRING) { - free(pItem->str); + taosMemoryFree(pItem->str); } - free(pItem->name); + taosMemoryFree(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -647,7 +647,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } taosCloseFile(&pFile); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); uInfo("load from cfg file %s success", filepath); return 0; diff --git a/source/util/src/tcrc32c.c b/source/util/src/tcrc32c.c index f7e1060be0c42fa3704c9a471e5ae5719bdf90e2..66e9240cd59aec61acb97a3284d37fa10a79698e 100644 --- a/source/util/src/tcrc32c.c +++ b/source/util/src/tcrc32c.c @@ -19,13 +19,13 @@ */ #define _DEFAULT_SOURCE -#include "tcrc32c.h" -#include "tdef.h" - #if !defined(_TD_ARM_) && !defined(_TD_MIPS_) #include #endif +#include "tcrc32c.h" +#include "tdef.h" + #define POLY 0x82f63b78 #define LONG_SHIFT 8192 #define SHORT_SHIFT 256 diff --git a/source/util/src/tdes.c b/source/util/src/tdes.c index a7f5131c2641cf84dd665b7357ad6ed38cefb880..0e0193a12369321008221d0637852c0a909c6685 100644 --- a/source/util/src/tdes.c +++ b/source/util/src/tdes.c @@ -47,7 +47,7 @@ char* taosDesImp(uint8_t* key, char* src, uint32_t len, int32_t process_mode) { uint8_t processed_block[9] = {0}; key_set key_sets[17]; memset(key_sets, 0, sizeof(key_sets)); - char* dest = calloc(len + 1, 1); + char* dest = taosMemoryCalloc(len + 1, 1); generate_sub_keys(key, key_sets); for (uint32_t block_count = 0; block_count < number_of_blocks; block_count++) { @@ -68,12 +68,12 @@ char* taosDesEncode(int64_t key, char* src, int32_t len) { char* taosDesDecode(int64_t key, char* src, int32_t len) { uint8_t* keyStr = (uint8_t*)(&key); - char* temp = calloc(len + 8, 1); + char* temp = taosMemoryCalloc(len + 8, 1); memcpy(temp, src, len); len += 8; char* decode = taosDesImp(keyStr, temp, len, DECRYPTION_MODE); - free(temp); + taosMemoryFree(temp); return decode; } diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index 94b4cced46adb7f72e89b27b415a52ccf48a0253..c40d5b02e6fcce760e63d9a2c756c6bf352ee258 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -44,7 +44,7 @@ void tCoderClear(SCoder* pCoder) { pNode = TD_SLIST_HEAD(&(pCoder->stack)); if (pNode == NULL) break; TD_SLIST_POP(&(pCoder->stack)); - free(pNode); + taosMemoryFree(pNode); } } @@ -55,7 +55,7 @@ int32_t tStartEncode(SCoder* pCoder) { if (pCoder->data) { if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1; - pNode = malloc(sizeof(*pNode)); + pNode = taosMemoryMalloc(sizeof(*pNode)); if (pNode == NULL) return -1; pNode->data = pCoder->data; @@ -93,7 +93,7 @@ void tEndEncode(SCoder* pCoder) { TD_CODER_MOVE_POS(pCoder, len); - free(pNode); + taosMemoryFree(pNode); } } @@ -104,7 +104,7 @@ int32_t tStartDecode(SCoder* pCoder) { ASSERT(pCoder->type == TD_DECODER); if (tDecodeI32(pCoder, &len) < 0) return -1; - pNode = malloc(sizeof(*pNode)); + pNode = taosMemoryMalloc(sizeof(*pNode)); if (pNode == NULL) return -1; pNode->data = pCoder->data; @@ -133,5 +133,5 @@ void tEndDecode(SCoder* pCoder) { pCoder->pos = pCoder->size + pNode->pos; pCoder->size = pNode->size; - free(pNode); + taosMemoryFree(pNode); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 469b944bf9bc6d3747430527770886b49deefe71..005995c8e2aec90fde5db0d079f8c61a4f50ead0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -84,6 +84,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") +TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM, "Out of Share memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") @@ -269,37 +270,22 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill // mnode-topic TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with aggregation is unsupported") +// mnode-sma +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "SMA does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma option") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OFFLINE, "Dnode is offline") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_READ_FILE_ERROR, "Read dnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR, "Write dnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED, "Mnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_NOT_DEPLOYED, "Mnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_INVALID_OPTION, "Mnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_READ_FILE_ERROR, "Read mnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR, "Write mnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED, "Qnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_NOT_DEPLOYED, "Qnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_INVALID_OPTION, "Qnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_READ_FILE_ERROR, "Read qnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR, "Write qnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED, "Snode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_NOT_DEPLOYED, "Snode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_INVALID_OPTION, "Snode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_READ_FILE_ERROR, "Read snode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR, "Write snode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED, "Bnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_NOT_DEPLOYED, "Bnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_INVALID_OPTION, "Bnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_READ_FILE_ERROR, "Read bnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR, "Write bnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_ALREADY_DEPLOYED, "Node already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_NOT_DEPLOYED, "Node not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_PARSE_FILE_ERROR, "Invalid json format") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_INVALID_OPTION, "Invalid node option") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED, "Vnode already deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_NOT_DEPLOYED, "Vnode not deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_INVALID_OPTION, "Vnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR, "Read vnodes.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR, "Write vnodes.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnodes") // vnode @@ -324,6 +310,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operat TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, "Database is syncing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TSDB_STATE, "Invalid tsdb state") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST, "SMA not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -349,8 +337,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") + // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") @@ -416,7 +407,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limi TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") // tfs -TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT, "tfs too many mount") TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY, "tfs duplicate primary mount") @@ -434,10 +425,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error" TAOS_DEFINE_ERROR(TSDB_CODE_CTG_SYS_ERROR, "catalog system error") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_DB_DROPPED, "Database is dropped") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_OUT_OF_SERVICE, "catalog is out of service") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_VG_META_MISMATCH, "table meta and vgroup mismatch") //scheduler TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") +TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order") #ifdef TAOS_ERROR_C }; @@ -455,14 +448,14 @@ static int32_t taosCompareTaosError(const void* a, const void* b) { return 0; } -static pthread_once_t tsErrorInit = PTHREAD_ONCE_INIT; +static TdThreadOnce tsErrorInit = PTHREAD_ONCE_INIT; static void tsSortError(void) { qsort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError); } const char* tstrerror(int32_t err) { - pthread_once(&tsErrorInit, tsSortError); + taosThreadOnce(&tsErrorInit, tsSortError); // this is a system errno if ((err & 0x00ff0000) == 0x00ff0000) { diff --git a/source/util/src/tfunctional.c b/source/util/src/tfunctional.c index c49fbdb50438effa0d980d50aaa00ea4a2d9428b..3b51d0046fa43222157eca76b509bf5696ef309c 100644 --- a/source/util/src/tfunctional.c +++ b/source/util/src/tfunctional.c @@ -17,21 +17,21 @@ #include "tfunctional.h" tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs) { - tGenericSavedFunc* pSavedFunc = malloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); + tGenericSavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs) { - tI32SavedFunc* pSavedFunc = malloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*)); + tI32SavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*)); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs) { - tVoidSavedFunc* pSavedFunc = malloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); + tVoidSavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 8b437b979728f9cd9a450028ffce8a64abae0fea..88e5c12770a373fda703fd4e75c3a13b8f6606f6 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -33,7 +33,7 @@ #define FREE_HASH_NODE(_n) \ do { \ - tfree(_n); \ + taosMemoryFreeClear(_n); \ } while (0); struct SHashNode { @@ -55,7 +55,7 @@ typedef struct SHashEntry { struct SHashObj { SHashEntry ** hashList; size_t capacity; // number of slots - size_t size; // number of elements in hash table + int64_t size; // number of elements in hash table _hash_fn_t hashFp; // hash function _equal_fn_t equalFp; // equal function _hash_free_fn_t freeFp; // hash node free callback function @@ -186,7 +186,7 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); - atomic_sub_fetch_32(&pNode->refCount, 1); + atomic_sub_fetch_16(&pNode->refCount, 1); if (prev != NULL) { prev->next = pNewNode; } else { @@ -238,7 +238,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp capacity = 4; } - SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); + SHashObj *pHashObj = (SHashObj *)taosMemoryCalloc(1, sizeof(SHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -254,26 +254,26 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHashEntry **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { - free(pHashObj); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); if (pHashObj->pMemBlock == NULL) { - free(pHashObj->hashList); - free(pHashObj); + taosMemoryFree(pHashObj->hashList); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + void *p = taosMemoryCalloc(pHashObj->capacity, sizeof(SHashEntry)); if (p == NULL) { taosArrayDestroy(pHashObj->pMemBlock); - free(pHashObj->hashList); - free(pHashObj); + taosMemoryFree(pHashObj->hashList); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -302,10 +302,10 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64(&pHashObj->size); + return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); } -int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) { if (pHashObj == NULL || key == NULL || keyLen == 0) { return -1; } @@ -440,14 +440,14 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** if (size != NULL) { if (*d == NULL) { *size = pNode->dataLen; - *d = calloc(1, *size); + *d = taosMemoryCalloc(1, *size); if (*d == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } } else if (*size < pNode->dataLen) { *size = pNode->dataLen; - char* tmp = realloc(*d, *size); + char* tmp = taosMemoryRealloc(*d, *size); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -508,7 +508,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pNode->removed == 0) { code = 0; // it is found - atomic_sub_fetch_32(&pNode->refCount, 1); + atomic_sub_fetch_16(&pNode->refCount, 1); pNode->removed = 1; if (pNode->refCount <= 0) { if (prevNode == NULL) { @@ -567,24 +567,24 @@ void taosHashClear(SHashObj *pHashObj) { taosHashWUnlock(pHashObj); } -// the input paras should be SHashObj **, so the origin input will be set by tfree(*pHashObj) +// the input paras should be SHashObj **, so the origin input will be set by taosMemoryFreeClear(*pHashObj) void taosHashCleanup(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } taosHashClear(pHashObj); - tfree(pHashObj->hashList); + taosMemoryFreeClear(pHashObj->hashList); // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); for (int32_t i = 0; i < memBlock; ++i) { void *p = taosArrayGetP(pHashObj->pMemBlock, i); - tfree(p); + taosMemoryFreeClear(p); } taosArrayDestroy(pHashObj->pMemBlock); - free(pHashObj); + taosMemoryFree(pHashObj); } // for profile only @@ -623,7 +623,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { // uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; @@ -632,7 +632,7 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->hashList = pNewEntryList; size_t inc = newCapacity - pHashObj->capacity; - void * p = calloc(inc, sizeof(SHashEntry)); + void * p = taosMemoryCalloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -683,7 +683,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHashNode *pNewNode = malloc(sizeof(SHashNode) + keyLen + dsize); + SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -755,7 +755,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { pNode = pNode->next; } - atomic_sub_fetch_32(&pOld->refCount, 1); + atomic_sub_fetch_16(&pOld->refCount, 1); if (pOld->refCount <=0) { if (prevNode) { prevNode->next = pOld->next; diff --git a/source/util/src/theap.c b/source/util/src/theap.c index 30af0483cc86c91b5ba2ba7a6fff81649f064a65..8c1a1db05709e3ef3eceb3329185d82f0b5485ed 100644 --- a/source/util/src/theap.c +++ b/source/util/src/theap.c @@ -19,7 +19,7 @@ size_t heapSize(Heap* heap) { return heap->nelts; } Heap* heapCreate(HeapCompareFn fn) { - Heap* heap = calloc(1, sizeof(Heap)); + Heap* heap = taosMemoryCalloc(1, sizeof(Heap)); if (heap == NULL) { return NULL; } @@ -30,7 +30,7 @@ Heap* heapCreate(HeapCompareFn fn) { return heap; } -void heapDestroy(Heap* heap) { free(heap); } +void heapDestroy(Heap* heap) { taosMemoryFree(heap); } HeapNode* heapMin(const Heap* heap) { return heap->min; } diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index 3ae537eae8a9b7c5b16bc91af5c994c140179ee4..b45113942c0fd22c5db95ea448af69780a317c04 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -18,12 +18,12 @@ #include "tlog.h" void *taosInitIdPool(int32_t maxId) { - id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); + id_pool_t *pIdPool = taosMemoryCalloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - pIdPool->freeList = calloc(maxId, sizeof(bool)); + pIdPool->freeList = taosMemoryCalloc(maxId, sizeof(bool)); if (pIdPool->freeList == NULL) { - free(pIdPool); + taosMemoryFree(pIdPool); return NULL; } @@ -31,7 +31,7 @@ void *taosInitIdPool(int32_t maxId) { pIdPool->numOfFree = maxId; pIdPool->freeSlot = 0; - pthread_mutex_init(&pIdPool->mutex, NULL); + taosThreadMutexInit(&pIdPool->mutex, NULL); uDebug("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); @@ -42,7 +42,7 @@ int32_t taosAllocateId(id_pool_t *pIdPool) { if (pIdPool == NULL) return -1; int32_t slot = -1; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); if (pIdPool->numOfFree > 0) { for (int32_t i = 0; i < pIdPool->maxId; ++i) { @@ -56,14 +56,14 @@ int32_t taosAllocateId(id_pool_t *pIdPool) { } } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return slot + 1; } void taosFreeId(id_pool_t *pIdPool, int32_t id) { if (pIdPool == NULL) return; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t slot = (id - 1) % pIdPool->maxId; if (pIdPool->freeList[slot]) { @@ -71,7 +71,7 @@ void taosFreeId(id_pool_t *pIdPool, int32_t id) { pIdPool->numOfFree++; } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); } void taosIdPoolCleanUp(id_pool_t *pIdPool) { @@ -79,26 +79,26 @@ void taosIdPoolCleanUp(id_pool_t *pIdPool) { uDebug("pool:%p is cleaned", pIdPool); - if (pIdPool->freeList) free(pIdPool->freeList); + if (pIdPool->freeList) taosMemoryFree(pIdPool->freeList); - pthread_mutex_destroy(&pIdPool->mutex); + taosThreadMutexDestroy(&pIdPool->mutex); memset(pIdPool, 0, sizeof(id_pool_t)); - free(pIdPool); + taosMemoryFree(pIdPool); } int32_t taosIdPoolNumOfUsed(id_pool_t *pIdPool) { - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t ret = pIdPool->maxId - pIdPool->numOfFree; - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } bool taosIdPoolMarkStatus(id_pool_t *pIdPool, int32_t id) { bool ret = false; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t slot = (id - 1) % pIdPool->maxId; if (!pIdPool->freeList[slot]) { @@ -109,7 +109,7 @@ bool taosIdPoolMarkStatus(id_pool_t *pIdPool, int32_t id) { ret = false; } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } @@ -118,12 +118,12 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { return 0; } - bool *idList = calloc(maxId, sizeof(bool)); + bool *idList = taosMemoryCalloc(maxId, sizeof(bool)); if (idList == NULL) { return -1; } - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); @@ -131,17 +131,17 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { bool *oldIdList = pIdPool->freeList; pIdPool->freeList = idList; - free(oldIdList); + taosMemoryFree(oldIdList); - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return 0; } int32_t taosIdPoolMaxSize(id_pool_t *pIdPool) { - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t ret = pIdPool->maxId; - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } \ No newline at end of file diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index a85da8cbbf56f6a69f582ecc1e34469786f7cc6d..0efcf517a9543730c01a86c19abced72eb3cf39a 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -255,7 +255,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v if (NULL == pJsonObj) { return TSDB_CODE_FAILED; } - *pObj = calloc(1, objSize); + *pObj = taosMemoryCalloc(1, objSize); if (NULL == *pObj) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 5fccba614b5c65beb6893ad0be64a61732bb200d..1d17b4a9e17aa7cafdd89ba273770e8751f09066 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -22,7 +22,7 @@ void tdListInit(SList *list, int32_t eleSize) { } SList *tdListNew(int32_t eleSize) { - SList *list = (SList *)malloc(sizeof(SList)); + SList *list = (SList *)taosMemoryMalloc(sizeof(SList)); if (list == NULL) return NULL; tdListInit(list, eleSize); @@ -33,14 +33,14 @@ void tdListEmpty(SList *list) { SListNode *node; while ((node = TD_DLIST_HEAD(list)) != NULL) { TD_DLIST_POP(list, node); - free(node); + taosMemoryFree(node); } } void *tdListFree(SList *list) { if (list) { tdListEmpty(list); - free(list); + taosMemoryFree(list); } return NULL; @@ -51,7 +51,7 @@ void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, no void tdListAppendNode(SList *list, SListNode *node) { TD_DLIST_APPEND(list, node); } int32_t tdListPrepend(SList *list, void *data) { - SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); + SListNode *node = (SListNode *)taosMemoryMalloc(sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); @@ -61,7 +61,7 @@ int32_t tdListPrepend(SList *list, void *data) { } int32_t tdListAppend(SList *list, void *data) { - SListNode *node = (SListNode *)calloc(1, sizeof(SListNode) + list->eleSize); + SListNode *node = (SListNode *)taosMemoryCalloc(1, sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 6dedc3f7406d3653c1513cb345b68cd17c966e1e..ef15f44f8fe5452f15dbe07f8e67228b98e45ddb 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -45,8 +45,8 @@ typedef struct { int32_t minBuffSize; TdFilePtr pFile; int32_t stop; - pthread_t asyncThread; - pthread_mutex_t buffMutex; + TdThread asyncThread; + TdThreadMutex buffMutex; tsem_t buffNotEmpty; } SLogBuff; @@ -59,7 +59,7 @@ typedef struct { pid_t pid; char logName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; - pthread_mutex_t logMutex; + TdThreadMutex logMutex; } SLogObj; static int8_t tsLogInited = 0; @@ -107,12 +107,12 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { - pthread_attr_t threadAttr; - pthread_attr_init(&threadAttr); - if (pthread_create(&(tsLogObj.logHandle->asyncThread), &threadAttr, taosAsyncOutputLog, tsLogObj.logHandle) != 0) { + TdThreadAttr threadAttr; + taosThreadAttrInit(&threadAttr); + if (taosThreadCreate(&(tsLogObj.logHandle->asyncThread), &threadAttr, taosAsyncOutputLog, tsLogObj.logHandle) != 0) { return -1; } - pthread_attr_destroy(&threadAttr); + taosThreadAttrDestroy(&threadAttr); return 0; } @@ -139,8 +139,9 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { - pthread_join(tsLogObj.logHandle->asyncThread, NULL); + taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); } + tsLogInited = 0; // In case that other threads still use log resources causing invalid write in valgrind // we comment two lines below. // taosLogBuffDestroy(tsLogObj.logHandle); @@ -223,22 +224,22 @@ static void *taosThreadToOpenNewFile(void *param) { } static int32_t taosOpenNewLogFile() { - pthread_mutex_lock(&tsLogObj.logMutex); + taosThreadMutexLock(&tsLogObj.logMutex); if (tsLogObj.lines > tsLogObj.maxLines && tsLogObj.openInProgress == 0) { tsLogObj.openInProgress = 1; uInfo("open new log file ......"); - pthread_t thread; - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + TdThread thread; + TdThreadAttr attr; + taosThreadAttrInit(&attr); + taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thread, &attr, taosThreadToOpenNewFile, NULL); - pthread_attr_destroy(&attr); + taosThreadCreate(&thread, &attr, taosThreadToOpenNewFile, NULL); + taosThreadAttrDestroy(&attr); } - pthread_mutex_unlock(&tsLogObj.logMutex); + taosThreadMutexUnlock(&tsLogObj.logMutex); return 0; } @@ -343,7 +344,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { char fileName[LOG_FILE_NAME_LEN + 50] = "\0"; sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag); - pthread_mutex_init(&tsLogObj.logMutex, NULL); + taosThreadMutexInit(&tsLogObj.logMutex, NULL); taosUmaskFile(0); tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); @@ -506,10 +507,10 @@ static void taosCloseLogByFd(TdFilePtr pFile) { static SLogBuff *taosLogBuffNew(int32_t bufSize) { SLogBuff *tLogBuff = NULL; - tLogBuff = calloc(1, sizeof(SLogBuff)); + tLogBuff = taosMemoryCalloc(1, sizeof(SLogBuff)); if (tLogBuff == NULL) return NULL; - LOG_BUF_BUFFER(tLogBuff) = malloc(bufSize); + LOG_BUF_BUFFER(tLogBuff) = taosMemoryMalloc(bufSize); if (LOG_BUF_BUFFER(tLogBuff) == NULL) goto _err; LOG_BUF_START(tLogBuff) = LOG_BUF_END(tLogBuff) = 0; @@ -517,14 +518,14 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { tLogBuff->minBuffSize = bufSize / 10; tLogBuff->stop = 0; - if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; + if (taosThreadMutexInit(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; // tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); return tLogBuff; _err: - tfree(LOG_BUF_BUFFER(tLogBuff)); - tfree(tLogBuff); + taosMemoryFreeClear(LOG_BUF_BUFFER(tLogBuff)); + taosMemoryFreeClear(tLogBuff); return NULL; } @@ -552,7 +553,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms if (tLogBuff == NULL || tLogBuff->stop) return -1; - pthread_mutex_lock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexLock(&LOG_BUF_MUTEX(tLogBuff)); start = LOG_BUF_START(tLogBuff); end = LOG_BUF_END(tLogBuff); @@ -566,7 +567,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms if (remainSize <= msgLen || ((lostLine > 0) && (remainSize <= (msgLen + tmpBufLen)))) { lostLine++; tsAsyncLogLostLines++; - pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexUnlock(&LOG_BUF_MUTEX(tLogBuff)); return -1; } @@ -587,7 +588,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms } */ - pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexUnlock(&LOG_BUF_MUTEX(tLogBuff)); return 0; } @@ -687,7 +688,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t compressSize = 163840; int32_t ret = 0; int32_t len = 0; - char *data = malloc(compressSize); + char *data = taosMemoryMalloc(compressSize); // gzFile dstFp = NULL; // srcFp = fopen(srcFileName, "r"); @@ -722,7 +723,7 @@ cmp_end: // if (dstFp) { // gzclose(dstFp); // } - free(data); + taosMemoryFree(data); return ret; } diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 6349ab170c3df30abd94f8b978f16aa598bc22da..aeb9ce310b3a8802120fa58e52a3fc88a57b7eec 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -36,7 +36,7 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, int32_t totalEntries = numOfSources << 1u; SMultiwayMergeTreeInfo* pTreeInfo = - (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); + (SMultiwayMergeTreeInfo*)taosMemoryCalloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); if (pTreeInfo == NULL) { uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); return TAOS_SYSTEM_ERROR(errno); @@ -76,7 +76,7 @@ void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree) { return; } - tfree(pTree); + taosMemoryFreeClear(pTree); } void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { diff --git a/source/util/src/tmallocator.c b/source/util/src/tmallocator.c index 057b908a583f297227982a8f101ab97a7ebe891a..0303af07e85306cac52a34746c2c80b9a16f649d 100644 --- a/source/util/src/tmallocator.c +++ b/source/util/src/tmallocator.c @@ -31,7 +31,7 @@ static size_t haUsage(SMemAllocator *pma); SMemAllocator *tdCreateHeapAllocator() { SMemAllocator *pma = NULL; - pma = calloc(1, sizeof(SMemAllocator) + sizeof(SHeapAllocator)); + pma = taosMemoryCalloc(1, sizeof(SMemAllocator) + sizeof(SHeapAllocator)); if (pma) { pma->impl = POINTER_SHIFT(pma, sizeof(SMemAllocator)); pma->malloc = haMalloc; @@ -53,7 +53,7 @@ static void *haMalloc(SMemAllocator *pma, size_t size) { size_t tsize = size + sizeof(size_t); SHeapAllocator *pha = (SHeapAllocator *)(pma->impl); - ptr = malloc(tsize); + ptr = taosMemoryMalloc(tsize); if (ptr) { *(size_t *)ptr = size; ptr = POINTER_SHIFT(ptr, sizeof(size_t)); @@ -97,7 +97,7 @@ static void haFree(SMemAllocator *pma, void *ptr) { /* TODO */ if (ptr) { size_t tsize = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t)) + sizeof(size_t); atomic_fetch_sub_64(&(pha->tusage), tsize); - free(POINTER_SHIFT(ptr, -sizeof(size_t))); + taosMemoryFree(POINTER_SHIFT(ptr, -sizeof(size_t))); } } diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index 1fc9bfc7ab458dfda7d1e74123bbea2366a70e38..7bf8e94de9d434f140fc9c35fb0f521486b22367 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -25,7 +25,7 @@ typedef struct { int32_t blockSize; /* block size in bytes */ int32_t *freeList; /* the index list */ char *pool; /* the actual mem block */ - pthread_mutex_t mutex; + TdThreadMutex mutex; } pool_t; mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { @@ -37,7 +37,7 @@ mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { return NULL; } - pool_p = (pool_t *)malloc(sizeof(pool_t)); + pool_p = (pool_t *)taosMemoryMalloc(sizeof(pool_t)); if (pool_p == NULL) { uError("mempool malloc failed\n"); return NULL; @@ -47,18 +47,18 @@ mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { pool_p->blockSize = blockSize; pool_p->numOfBlock = numOfBlock; - pool_p->pool = (char *)malloc((size_t)(blockSize * numOfBlock)); - pool_p->freeList = (int32_t *)malloc(sizeof(int32_t) * (size_t)numOfBlock); + pool_p->pool = (char *)taosMemoryMalloc((size_t)(blockSize * numOfBlock)); + pool_p->freeList = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * (size_t)numOfBlock); if (pool_p->pool == NULL || pool_p->freeList == NULL) { uError("failed to allocate memory\n"); - tfree(pool_p->freeList); - tfree(pool_p->pool); - tfree(pool_p); + taosMemoryFreeClear(pool_p->freeList); + taosMemoryFreeClear(pool_p->pool); + taosMemoryFreeClear(pool_p); return NULL; } - pthread_mutex_init(&(pool_p->mutex), NULL); + taosThreadMutexInit(&(pool_p->mutex), NULL); memset(pool_p->pool, 0, (size_t)(blockSize * numOfBlock)); for (i = 0; i < pool_p->numOfBlock; ++i) pool_p->freeList[i] = i; @@ -73,7 +73,7 @@ char *taosMemPoolMalloc(mpool_h handle) { char *pos = NULL; pool_t *pool_p = (pool_t *)handle; - pthread_mutex_lock(&(pool_p->mutex)); + taosThreadMutexLock(&(pool_p->mutex)); if (pool_p->numOfFree > 0) { pos = pool_p->pool + pool_p->blockSize * (pool_p->freeList[pool_p->first]); @@ -82,7 +82,7 @@ char *taosMemPoolMalloc(mpool_h handle) { pool_p->numOfFree--; } - pthread_mutex_unlock(&(pool_p->mutex)); + taosThreadMutexUnlock(&(pool_p->mutex)); if (pos == NULL) uDebug("mempool: out of memory"); return pos; @@ -108,20 +108,20 @@ void taosMemPoolFree(mpool_h handle, char *pMem) { memset(pMem, 0, (size_t)pool_p->blockSize); - pthread_mutex_lock(&pool_p->mutex); + taosThreadMutexLock(&pool_p->mutex); pool_p->freeList[(pool_p->first + pool_p->numOfFree) % pool_p->numOfBlock] = index; pool_p->numOfFree++; - pthread_mutex_unlock(&pool_p->mutex); + taosThreadMutexUnlock(&pool_p->mutex); } void taosMemPoolCleanUp(mpool_h handle) { pool_t *pool_p = (pool_t *)handle; - pthread_mutex_destroy(&pool_p->mutex); - if (pool_p->pool) free(pool_p->pool); - if (pool_p->freeList) free(pool_p->freeList); + taosThreadMutexDestroy(&pool_p->mutex); + if (pool_p->pool) taosMemoryFree(pool_p->pool); + if (pool_p->freeList) taosMemoryFree(pool_p->freeList); memset(pool_p, 0, sizeof(*pool_p)); - free(pool_p); + taosMemoryFree(pool_p); } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index a9d925ac076b3345e5abf99a6e598c112c2a251f..d834263b940ae6932bcd121b5a18791a491012d5 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -266,7 +266,7 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag pBuf->numOfPages += 1; - SPageInfo* ppi = malloc(sizeof(SPageInfo)); + SPageInfo* ppi = taosMemoryMalloc(sizeof(SPageInfo)); ppi->pageId = pageId; ppi->pData = NULL; @@ -330,7 +330,7 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { assert(d->pn == pn); d->pn = NULL; - tfree(pn); + taosMemoryFreeClear(pn); bufPage = flushPageToDisk(pBuf, d); } @@ -359,7 +359,7 @@ static SPageInfo* getPageInfoFromPayload(void* page) { int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir) { - *pBuf = calloc(1, sizeof(SDiskbasedBuf)); + *pBuf = taosMemoryCalloc(1, sizeof(SDiskbasedBuf)); SDiskbasedBuf* pPBuf = *pBuf; if (pPBuf == NULL) { @@ -386,7 +386,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem // init id hash table _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); pPBuf->groupSet = taosHashInit(10, fn, true, false); - pPBuf->assistBuf = malloc(pPBuf->pageSize + 2); // EXTRA BYTES + pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2); // EXTRA BYTES pPBuf->all = taosHashInit(10, fn, true, false); char path[PATH_MAX] = {0}; @@ -422,7 +422,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { pi = *(SPageInfo**)pItem->data; pi->used = true; *pageId = pi->pageId; - tfree(pItem); + taosMemoryFreeClear(pItem); } else { // create a new pageinfo // register new id in this group *pageId = (++pBuf->allocateId); @@ -441,7 +441,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { // allocate buf if (availablePage == NULL) { - pi->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. + pi->pData = taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. } else { pi->pData = availablePage; } @@ -483,7 +483,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { } if (availablePage == NULL) { - (*pi)->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); + (*pi)->pData = taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize)); } else { (*pi)->pData = availablePage; } @@ -564,15 +564,15 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { } taosRemoveFile(pBuf->path); - tfree(pBuf->path); + taosMemoryFreeClear(pBuf->path); SArray** p = taosHashIterate(pBuf->groupSet, NULL); while (p) { size_t n = taosArrayGetSize(*p); for (int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); - tfree(pi->pData); - tfree(pi); + taosMemoryFreeClear(pi->pData); + taosMemoryFreeClear(pi); } taosArrayDestroy(*p); @@ -588,9 +588,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { taosHashCleanup(pBuf->groupSet); taosHashCleanup(pBuf->all); - tfree(pBuf->id); - tfree(pBuf->assistBuf); - tfree(pBuf); + taosMemoryFreeClear(pBuf->id); + taosMemoryFreeClear(pBuf->assistBuf); + taosMemoryFreeClear(pBuf); } SPageInfo* getLastPageInfo(SIDList pList) { @@ -625,8 +625,8 @@ void dBufSetBufPageRecycled(SDiskbasedBuf* pBuf, void* pPage) { // add this pageinfo into the free page info list SListNode* pNode = tdListPopNode(pBuf->lruList, ppi->pn); - tfree(ppi->pData); - tfree(pNode); + taosMemoryFreeClear(ppi->pData); + taosMemoryFreeClear(pNode); tdListAppend(pBuf->freePgList, &ppi); } diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c new file mode 100644 index 0000000000000000000000000000000000000000..9e55df44cfbeb225ad2b1a521414573cfe117dc3 --- /dev/null +++ b/source/util/src/tprocess.c @@ -0,0 +1,460 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "tprocess.h" +#include "taoserror.h" +#include "tlog.h" +#include "tqueue.h" + +// todo +#include +#include + +#define SHM_DEFAULT_SIZE (20 * 1024 * 1024) +#define CEIL8(n) (ceil((float)(n) / 8) * 8) +typedef void *(*ProcThreadFp)(void *param); + +typedef struct SProcQueue { + int32_t head; + int32_t tail; + int32_t total; + int32_t avail; + int32_t items; + char *pBuffer; + ProcMallocFp mallocHeadFp; + ProcFreeFp freeHeadFp; + ProcMallocFp mallocBodyFp; + ProcFreeFp freeBodyFp; + ProcConsumeFp consumeFp; + void *pParent; + tsem_t sem; + TdThreadMutex *mutex; + int32_t mutexShmid; + int32_t bufferShmid; + const char *name; +} SProcQueue; + +typedef struct SProcObj { + TdThread childThread; + SProcQueue *pChildQueue; + TdThread parentThread; + SProcQueue *pParentQueue; + const char *name; + int32_t pid; + bool isChild; + bool stopFlag; + bool testFlag; +} SProcObj; + +static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { + TdThreadMutex *pMutex = NULL; + TdThreadMutexAttr mattr = {0}; + int32_t shmid = -1; + int32_t code = -1; + + if (pthread_mutexattr_init(&mattr) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while init attr since %s", terrstr()); + goto _OVER; + } + + if (pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while set shared since %s", terrstr()); + goto _OVER; + } + + shmid = shmget(IPC_PRIVATE, sizeof(TdThreadMutex), 0600); + if (shmid <= 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while shmget since %s", terrstr()); + goto _OVER; + } + + pMutex = (TdThreadMutex *)shmat(shmid, NULL, 0); + if (pMutex == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while shmat since %s", terrstr()); + goto _OVER; + } + + if (taosThreadMutexInit(pMutex, &mattr) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex since %s", terrstr()); + goto _OVER; + } + + code = 0; + +_OVER: + if (code != 0) { + taosThreadMutexDestroy(pMutex); + shmctl(shmid, IPC_RMID, NULL); + } else { + *ppMutex = pMutex; + *pShmid = shmid; + } + + pthread_mutexattr_destroy(&mattr); + return code; +} + +static void taosProcDestroyMutex(TdThreadMutex *pMutex, int32_t *pShmid) { + if (pMutex != NULL) { + taosThreadMutexDestroy(pMutex); + } + if (*pShmid > 0) { + shmctl(*pShmid, IPC_RMID, NULL); + } +} + +static int32_t taosProcInitBuffer(void **ppBuffer, int32_t size) { + int32_t shmid = shmget(IPC_PRIVATE, size, IPC_CREAT | 0600); + if (shmid <= 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init buffer while shmget since %s", terrstr()); + return -1; + } + + void *shmptr = shmat(shmid, NULL, 0); + if (shmptr == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init buffer while shmat since %s", terrstr()); + shmctl(shmid, IPC_RMID, NULL); + return -1; + } + + *ppBuffer = shmptr; + return shmid; +} + +static void taosProcDestroyBuffer(void *pBuffer, int32_t *pShmid) { + if (*pShmid > 0) { + shmctl(*pShmid, IPC_RMID, NULL); + } +} + +static SProcQueue *taosProcQueueInit(int32_t size) { + if (size <= 0) size = SHM_DEFAULT_SIZE; + + int32_t bufSize = CEIL8(size); + int32_t headSize = CEIL8(sizeof(SProcQueue)); + + SProcQueue *pQueue = NULL; + int32_t shmId = taosProcInitBuffer((void **)&pQueue, bufSize + headSize); + if (shmId <= 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pQueue->bufferShmid = shmId; + + if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { + taosMemoryFree(pQueue); + return NULL; + } + + if (tsem_init(&pQueue->sem, 1, 0) != 0) { + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + taosMemoryFree(pQueue); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + tsem_destroy(&pQueue->sem); + taosMemoryFree(pQueue); + return NULL; + } + + pQueue->head = 0; + pQueue->tail = 0; + pQueue->total = bufSize; + pQueue->avail = bufSize; + pQueue->items = 0; + pQueue->pBuffer = (char *)pQueue + headSize; + return pQueue; +} + +static void taosProcQueueCleanup(SProcQueue *pQueue) { + if (pQueue != NULL) { + uDebug("proc:%s, queue:%p clean up", pQueue->name, pQueue); + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + tsem_destroy(&pQueue->sem); + taosMemoryFree(pQueue); + } +} + +static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHeadLen, char *pBody, int32_t rawBodyLen) { + const int32_t headLen = CEIL8(rawHeadLen); + const int32_t bodyLen = CEIL8(rawBodyLen); + const int32_t fullLen = headLen + bodyLen + 8; + + taosThreadMutexLock(pQueue->mutex); + if (fullLen > pQueue->avail) { + taosThreadMutexUnlock(pQueue->mutex); + terrno = TSDB_CODE_OUT_OF_SHM_MEM; + return -1; + } + + if (pQueue->tail < pQueue->total) { + *(int32_t *)(pQueue->pBuffer + pQueue->head) = headLen; + *(int32_t *)(pQueue->pBuffer + pQueue->head + 4) = bodyLen; + } else { + *(int32_t *)(pQueue->pBuffer) = headLen; + *(int32_t *)(pQueue->pBuffer + 4) = bodyLen; + } + + if (pQueue->tail < pQueue->head) { + memcpy(pQueue->pBuffer + pQueue->tail + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->tail + 8 + headLen, pBody, rawBodyLen); + pQueue->tail = pQueue->tail + 8 + headLen + bodyLen; + } else { + int32_t remain = pQueue->total - pQueue->tail; + if (remain == 0) { + memcpy(pQueue->pBuffer + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + 8 + headLen, pBody, rawBodyLen); + pQueue->tail = 8 + headLen + bodyLen; + } else if (remain == 8) { + memcpy(pQueue->pBuffer, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + headLen, pBody, rawBodyLen); + pQueue->tail = headLen + bodyLen; + } else if (remain < 8 + headLen) { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, remain - 8); + memcpy(pQueue->pBuffer, pHead + remain - 8, rawHeadLen - (remain - 8)); + memcpy(pQueue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); + pQueue->tail = headLen - (remain - 8) + bodyLen; + } else if (remain < 8 + bodyLen) { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->head + 8 + headLen, pBody, remain - 8 - headLen); + memcpy(pQueue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); + pQueue->tail = bodyLen - (remain - 8 - headLen); + } else { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->head + headLen + 8, pBody, rawBodyLen); + pQueue->tail = pQueue->head + headLen + bodyLen + 8; + } + } + + pQueue->avail -= fullLen; + pQueue->items++; + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + + uTrace("proc:%s, push msg:%p:%d cont:%p:%d to queue:%p", pQueue->name, pHead, rawHeadLen, pBody, rawBodyLen, pQueue); + return 0; +} + +static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHeadLen, void **ppBody, + int32_t *pBodyLen) { + tsem_wait(&pQueue->sem); + + taosThreadMutexLock(pQueue->mutex); + if (pQueue->total - pQueue->avail <= 0) { + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + terrno = TSDB_CODE_OUT_OF_SHM_MEM; + return -1; + } + + int32_t headLen = 0; + int32_t bodyLen = 0; + if (pQueue->head < pQueue->total) { + headLen = *(int32_t *)(pQueue->pBuffer + pQueue->head); + bodyLen = *(int32_t *)(pQueue->pBuffer + pQueue->head + 4); + } else { + headLen = *(int32_t *)(pQueue->pBuffer); + bodyLen = *(int32_t *)(pQueue->pBuffer + 4); + } + + void *pHead = (*pQueue->mallocHeadFp)(headLen); + void *pBody = (*pQueue->mallocBodyFp)(bodyLen); + if (pHead == NULL || pBody == NULL) { + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + (*pQueue->freeHeadFp)(pHead); + (*pQueue->freeBodyFp)(pBody); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (pQueue->head < pQueue->tail) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, bodyLen); + pQueue->head = pQueue->head + 8 + headLen + bodyLen; + } else { + int32_t remain = pQueue->total - pQueue->head; + if (remain == 0) { + memcpy(pHead, pQueue->pBuffer + 8, headLen); + memcpy(pBody, pQueue->pBuffer + 8 + headLen, bodyLen); + pQueue->head = 8 + headLen + bodyLen; + } else if (remain == 8) { + memcpy(pHead, pQueue->pBuffer, headLen); + memcpy(pBody, pQueue->pBuffer + headLen, bodyLen); + pQueue->head = headLen + bodyLen; + } else if (remain < 8 + headLen) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, remain - 8); + memcpy(pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); + memcpy(pBody, pQueue->pBuffer + headLen - (remain - 8), bodyLen); + pQueue->head = headLen - (remain - 8) + bodyLen; + } else if (remain < 8 + bodyLen) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, remain - 8 - headLen); + memcpy(pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); + pQueue->head = bodyLen - (remain - 8 - headLen); + } else { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + headLen + 8, bodyLen); + pQueue->head = pQueue->head + headLen + bodyLen + 8; + } + } + + pQueue->avail = pQueue->avail + headLen + bodyLen + 8; + pQueue->items--; + taosThreadMutexUnlock(pQueue->mutex); + + *ppHead = pHead; + *ppBody = pBody; + *pHeadLen = headLen; + *pBodyLen = bodyLen; + + uTrace("proc:%s, get msg:%p:%d cont:%p:%d from queue:%p", pQueue->name, pHead, headLen, pBody, bodyLen, pQueue); + return 0; +} + +SProcObj *taosProcInit(const SProcCfg *pCfg) { + SProcObj *pProc = taosMemoryCalloc(1, sizeof(SProcObj)); + if (pProc == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pProc->name = pCfg->name; + pProc->testFlag = pCfg->testFlag; + + pProc->pChildQueue = taosProcQueueInit(pCfg->childQueueSize); + pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); + if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { + taosProcQueueCleanup(pProc->pChildQueue); + taosMemoryFree(pProc); + return NULL; + } + + pProc->pChildQueue->name = pCfg->name; + pProc->pChildQueue->pParent = pCfg->pParent; + pProc->pChildQueue->mallocHeadFp = pCfg->childMallocHeadFp; + pProc->pChildQueue->freeHeadFp = pCfg->childFreeHeadFp; + pProc->pChildQueue->mallocBodyFp = pCfg->childMallocBodyFp; + pProc->pChildQueue->freeBodyFp = pCfg->childFreeBodyFp; + pProc->pChildQueue->consumeFp = pCfg->childConsumeFp; + pProc->pParentQueue->name = pCfg->name; + pProc->pParentQueue->pParent = pCfg->pParent; + pProc->pParentQueue->mallocHeadFp = pCfg->parentdMallocHeadFp; + pProc->pParentQueue->freeHeadFp = pCfg->parentFreeHeadFp; + pProc->pParentQueue->mallocBodyFp = pCfg->parentMallocBodyFp; + pProc->pParentQueue->freeBodyFp = pCfg->parentFreeBodyFp; + pProc->pParentQueue->consumeFp = pCfg->parentConsumeFp; + + uDebug("proc:%s, initialized, child queue:%p parent queue:%p", pProc->name, pProc->pChildQueue, pProc->pParentQueue); + + if (!pProc->testFlag) { + pProc->pid = fork(); + if (pProc->pid == 0) { + pProc->isChild = 1; + uInfo("this is child process, pid:%d", pProc->pid); + } else { + pProc->isChild = 0; + uInfo("this is parent process, pid:%d", pProc->pid); + } + } + + return pProc; +} + +static void taosProcThreadLoop(SProcQueue *pQueue) { + ProcConsumeFp consumeFp = pQueue->consumeFp; + void *pParent = pQueue->pParent; + void *pHead, *pBody; + int32_t headLen, bodyLen; + + uDebug("proc:%s, start to get message from queue:%p", pQueue->name, pQueue); + + while (1) { + int32_t code = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen); + if (code < 0) { + uDebug("proc:%s, get no message from queue:%p and exiting", pQueue->name, pQueue); + break; + } else if (code < 0) { + uTrace("proc:%s, get no message from queue:%p since %s", pQueue->name, pQueue, terrstr()); + taosMsleep(1); + continue; + } else { + (*consumeFp)(pParent, pHead, headLen, pBody, bodyLen); + } + } +} + +int32_t taosProcRun(SProcObj *pProc) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pProc->isChild || pProc->testFlag) { + if (taosThreadCreate(&pProc->childThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pChildQueue) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create thread since %s", terrstr()); + return -1; + } + uDebug("proc:%s, child start to consume queue:%p", pProc->name, pProc->pChildQueue); + } + + if (!pProc->isChild || pProc->testFlag) { + if (taosThreadCreate(&pProc->parentThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pParentQueue) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create thread since %s", terrstr()); + return -1; + } + uDebug("proc:%s, parent start to consume queue:%p", pProc->name, pProc->pParentQueue); + } + + return 0; +} + +void taosProcStop(SProcObj *pProc) { + pProc->stopFlag = true; + // todo join +} + +bool taosProcIsChild(SProcObj *pProc) { return pProc->isChild; } + +void taosProcCleanup(SProcObj *pProc) { + if (pProc != NULL) { + uDebug("proc:%s, clean up", pProc->name); + taosProcStop(pProc); + taosProcQueueCleanup(pProc->pChildQueue); + taosProcQueueCleanup(pProc->pParentQueue); + taosMemoryFree(pProc); + } +} + +int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) { + return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen); +} + +int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) { + return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen); +} diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 000352203348ec8ea0ff3d587c8de89557d994dd..b01e1ea1daccd3679dcb5117219ec153c9716182 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -37,13 +37,13 @@ typedef struct STaosQueue { void *ahandle; // for queue set FItem itemFp; FItems itemsFp; - pthread_mutex_t mutex; + TdThreadMutex mutex; } STaosQueue; typedef struct STaosQset { STaosQueue *head; STaosQueue *current; - pthread_mutex_t mutex; + TdThreadMutex mutex; int32_t numOfQueues; int32_t numOfItems; tsem_t sem; @@ -57,18 +57,17 @@ typedef struct STaosQall { } STaosQall; STaosQueue *taosOpenQueue() { - STaosQueue *queue = calloc(1, sizeof(STaosQueue)); + STaosQueue *queue = taosMemoryCalloc(1, sizeof(STaosQueue)); if (queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - if (pthread_mutex_init(&queue->mutex, NULL) != 0) { + if (taosThreadMutexInit(&queue->mutex, NULL) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - queue->threadId = -1; uDebug("queue:%p is opened", queue); return queue; } @@ -84,11 +83,11 @@ void taosCloseQueue(STaosQueue *queue) { STaosQnode *pTemp; STaosQset *qset; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); STaosQnode *pNode = queue->head; queue->head = NULL; qset = queue->qset; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (queue->qset) { taosRemoveFromQset(qset, queue); @@ -97,11 +96,11 @@ void taosCloseQueue(STaosQueue *queue) { while (pNode) { pTemp = pNode; pNode = pNode->next; - free(pTemp); + taosMemoryFree(pTemp); } - pthread_mutex_destroy(&queue->mutex); - free(queue); + taosThreadMutexDestroy(&queue->mutex); + taosMemoryFree(queue); uDebug("queue:%p is closed", queue); } @@ -110,24 +109,24 @@ bool taosQueueEmpty(STaosQueue *queue) { if (queue == NULL) return true; bool empty = false; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head == NULL && queue->tail == NULL) { empty = true; } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return empty; } int32_t taosQueueSize(STaosQueue *queue) { - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); int32_t numOfItems = queue->numOfItems; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return numOfItems; } void *taosAllocateQitem(int32_t size) { - STaosQnode *pNode = calloc(1, sizeof(STaosQnode) + size); + STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -144,14 +143,14 @@ void taosFreeQitem(void *pItem) { char *temp = pItem; temp -= sizeof(STaosQnode); uTrace("item:%p, node:%p is freed", pItem, temp); - free(temp); + taosMemoryFree(temp); } int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { STaosQnode *pNode = (STaosQnode *)(((char *)pItem) - sizeof(STaosQnode)); pNode->next = NULL; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->tail) { queue->tail->next = pNode; @@ -165,7 +164,7 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); uTrace("item:%p is put into queue:%p, items:%d", pItem, queue, queue->numOfItems); - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (queue->qset) tsem_post(&queue->qset->sem); @@ -176,7 +175,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { STaosQnode *pNode = NULL; int32_t code = 0; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { pNode = queue->head; @@ -189,20 +188,20 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return code; } -STaosQall *taosAllocateQall() { return calloc(1, sizeof(STaosQall)); } +STaosQall *taosAllocateQall() { return taosMemoryCalloc(1, sizeof(STaosQall)); } -void taosFreeQall(STaosQall *qall) { free(qall); } +void taosFreeQall(STaosQall *qall) { taosMemoryFree(qall); } int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { int32_t code = 0; bool empty; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); empty = queue->head == NULL; if (!empty) { @@ -219,7 +218,7 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, qall->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); // if source queue is empty, we set destination qall to empty too. if (empty) { @@ -249,13 +248,13 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } STaosQset *taosOpenQset() { - STaosQset *qset = calloc(sizeof(STaosQset), 1); + STaosQset *qset = taosMemoryCalloc(sizeof(STaosQset), 1); if (qset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pthread_mutex_init(&qset->mutex, NULL); + taosThreadMutexInit(&qset->mutex, NULL); tsem_init(&qset->sem, 0, 0); uDebug("qset:%p is opened", qset); @@ -266,7 +265,7 @@ void taosCloseQset(STaosQset *qset) { if (qset == NULL) return; // remove all the queues from qset - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); while (qset->head) { STaosQueue *queue = qset->head; qset->head = qset->head->next; @@ -274,11 +273,11 @@ void taosCloseQset(STaosQset *qset) { queue->qset = NULL; queue->next = NULL; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); - pthread_mutex_destroy(&qset->mutex); + taosThreadMutexDestroy(&qset->mutex); tsem_destroy(&qset->sem); - free(qset); + taosMemoryFree(qset); uDebug("qset:%p is closed", qset); } @@ -293,19 +292,19 @@ void taosQsetThreadResume(STaosQset *qset) { int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { if (queue->qset) return -1; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); queue->next = qset->head; queue->ahandle = ahandle; qset->head = queue; qset->numOfQueues++; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); atomic_add_fetch_32(&qset->numOfItems, queue->numOfItems); queue->qset = qset; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); uTrace("queue:%p is added into qset:%p", queue, qset); return 0; @@ -314,7 +313,7 @@ int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { STaosQueue *tqueue = NULL; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); if (qset->head) { if (qset->head == queue) { @@ -339,15 +338,15 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { if (qset->current == queue) qset->current = tqueue->next; qset->numOfQueues--; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); atomic_sub_fetch_32(&qset->numOfItems, queue->numOfItems); queue->qset = NULL; queue->next = NULL; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); } } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); uDebug("queue:%p is removed from qset:%p", queue, qset); } @@ -360,7 +359,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI tsem_wait(&qset->sem); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; @@ -369,7 +368,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI if (queue == NULL) break; if (queue->head == NULL) continue; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { pNode = queue->head; @@ -385,11 +384,11 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (pNode) break; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return code; } @@ -399,7 +398,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand int32_t code = 0; tsem_wait(&qset->sem); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; @@ -408,7 +407,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand if (queue == NULL) break; if (queue->head == NULL) continue; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { qall->current = queue->head; @@ -428,59 +427,12 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand } } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (code != 0) break; } - pthread_mutex_unlock(&qset->mutex); - return code; -} - -int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId) { - STaosQnode *pNode = NULL; - int32_t code = -1; - - tsem_wait(&qset->sem); - - pthread_mutex_lock(&qset->mutex); - - for (int32_t i = 0; i < qset->numOfQueues; ++i) { - if (qset->current == NULL) qset->current = qset->head; - STaosQueue *queue = qset->current; - if (queue) qset->current = queue->next; - if (queue == NULL) break; - if (queue->head == NULL) continue; - if (queue->threadId != -1 && queue->threadId != threadId) { - code = 0; - continue; - } - - pthread_mutex_lock(&queue->mutex); - - if (queue->head) { - pNode = queue->head; - pNode->queue = queue; - queue->threadId = threadId; - *ppItem = pNode->item; - - if (ahandle) *ahandle = queue->ahandle; - if (itemFp) *itemFp = queue->itemFp; - - queue->head = pNode->next; - if (queue->head == NULL) queue->tail = NULL; - queue->numOfItems--; - atomic_sub_fetch_32(&qset->numOfItems, 1); - code = 1; - uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); - } - - pthread_mutex_unlock(&queue->mutex); - if (pNode) break; - } - - pthread_mutex_unlock(&qset->mutex); - + taosThreadMutexUnlock(&qset->mutex); return code; } @@ -488,21 +440,20 @@ void taosResetQsetThread(STaosQset *qset, void *pItem) { if (pItem == NULL) return; STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); - pthread_mutex_lock(&qset->mutex); - pNode->queue->threadId = -1; + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < pNode->queue->numOfItems; ++i) { tsem_post(&qset->sem); } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); } int32_t taosGetQueueItemsNumber(STaosQueue *queue) { if (!queue) return 0; int32_t num; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); num = queue->numOfItems; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return num; } @@ -510,8 +461,8 @@ int32_t taosGetQsetItemsNumber(STaosQset *qset) { if (!qset) return 0; int32_t num = 0; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); num = qset->numOfItems; - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return num; } diff --git a/source/util/src/tref.c b/source/util/src/tref.c index a9f6c21bf80ad48f34c895e25c46074fe013d7cd..2e4c33bc8758f995c10e8efef6569e9f30cb67ee 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -45,8 +45,8 @@ typedef struct { } SRefSet; static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; -static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t tsRefMutex; +static TdThreadOnce tsRefModuleInit = PTHREAD_ONCE_INIT; +static TdThreadMutex tsRefMutex; static int32_t tsRefSetNum = 0; static int32_t tsNextId = 0; @@ -63,22 +63,22 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { int64_t *lockedBy; int32_t i, rsetId; - pthread_once(&tsRefModuleInit, taosInitRefModule); + taosThreadOnce(&tsRefModuleInit, taosInitRefModule); - nodeList = calloc(sizeof(SRefNode *), (size_t)max); + nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; return -1; } - lockedBy = calloc(sizeof(int64_t), (size_t)max); + lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max); if (lockedBy == NULL) { - free(nodeList); + taosMemoryFree(nodeList); terrno = TSDB_CODE_REF_NO_MEMORY; return -1; } - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; @@ -102,12 +102,12 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum); } else { rsetId = TSDB_CODE_REF_FULL; - free(nodeList); - free(lockedBy); + taosMemoryFree(nodeList); + taosMemoryFree(lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); return rsetId; } @@ -124,7 +124,7 @@ int32_t taosCloseRef(int32_t rsetId) { pSet = tsRefSetList + rsetId; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); if (pSet->state == TSDB_REF_STATE_ACTIVE) { pSet->state = TSDB_REF_STATE_DELETED; @@ -134,7 +134,7 @@ int32_t taosCloseRef(int32_t rsetId) { uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); if (deleted) taosDecRsetCount(pSet); @@ -162,7 +162,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) { return -1; } - pNode = calloc(sizeof(SRefNode), 1); + pNode = taosMemoryCalloc(sizeof(SRefNode), 1); if (pNode == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; return -1; @@ -354,7 +354,7 @@ int32_t taosListRef() { SRefNode *pNode; int32_t num = 0; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); for (int32_t i = 0; i < TSDB_REF_OBJECTS; ++i) { pSet = tsRefSetList + i; @@ -374,7 +374,7 @@ int32_t taosListRef() { } } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); return num; } @@ -445,7 +445,7 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); (*pSet->fp)(pNode->p); - free(pNode); + taosMemoryFree(pNode); taosDecRsetCount(pSet); } @@ -470,7 +470,7 @@ static void taosUnlockList(int64_t *lockedBy) { } } -static void taosInitRefModule(void) { pthread_mutex_init(&tsRefMutex, NULL); } +static void taosInitRefModule(void) { taosThreadMutexInit(&tsRefMutex, NULL); } static void taosIncRsetCount(SRefSet *pSet) { atomic_add_fetch_32(&pSet->count, 1); @@ -483,19 +483,19 @@ static void taosDecRsetCount(SRefSet *pSet) { if (count > 0) return; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); if (pSet->state != TSDB_REF_STATE_EMPTY) { pSet->state = TSDB_REF_STATE_EMPTY; pSet->max = 0; pSet->fp = NULL; - tfree(pSet->nodeList); - tfree(pSet->lockedBy); + taosMemoryFreeClear(pSet->nodeList); + taosMemoryFreeClear(pSet->lockedBy); tsRefSetNum--; uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); } diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 740e742bad5e9b9f9e50194bcc02aadc1146cba3..2deba5077b37e227b507b4d982dd8d7883de1199 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -26,12 +26,12 @@ typedef struct { char label[TSDB_LABEL_LEN]; tsem_t emptySem; tsem_t fullSem; - pthread_mutex_t queueMutex; + TdThreadMutex queueMutex; int32_t fullSlot; int32_t emptySlot; int32_t queueSize; int32_t numOfThreads; - pthread_t *qthread; + TdThread *qthread; SSchedMsg *queue; bool stop; void *pTmrCtrl; @@ -42,20 +42,20 @@ static void *taosProcessSchedQueue(void *param); static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *label) { - SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1); + SSchedQueue *pSched = (SSchedQueue *)taosMemoryCalloc(sizeof(SSchedQueue), 1); if (pSched == NULL) { uError("%s: no enough memory for pSched", label); return NULL; } - pSched->queue = (SSchedMsg *)calloc(sizeof(SSchedMsg), queueSize); + pSched->queue = (SSchedMsg *)taosMemoryCalloc(sizeof(SSchedMsg), queueSize); if (pSched->queue == NULL) { uError("%s: no enough memory for queue", label); taosCleanUpScheduler(pSched); return NULL; } - pSched->qthread = calloc(sizeof(pthread_t), numOfThreads); + pSched->qthread = taosMemoryCalloc(sizeof(TdThread), numOfThreads); if (pSched->qthread == NULL) { uError("%s: no enough memory for qthread", label); taosCleanUpScheduler(pSched); @@ -68,7 +68,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab pSched->fullSlot = 0; pSched->emptySlot = 0; - if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) { + if (taosThreadMutexInit(&pSched->queueMutex, NULL) < 0) { uError("init %s:queueMutex failed(%s)", label, strerror(errno)); taosCleanUpScheduler(pSched); return NULL; @@ -88,11 +88,11 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab pSched->stop = false; for (int32_t i = 0; i < numOfThreads; ++i) { - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int32_t code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); - pthread_attr_destroy(&attr); + TdThreadAttr attr; + taosThreadAttrInit(&attr); + taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_JOINABLE); + int32_t code = taosThreadCreate(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); + taosThreadAttrDestroy(&attr); if (code != 0) { uError("%s: failed to create rpc thread(%s)", label, strerror(errno)); taosCleanUpScheduler(pSched); @@ -135,7 +135,7 @@ void *taosProcessSchedQueue(void *scheduler) { break; } - if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -144,7 +144,7 @@ void *taosProcessSchedQueue(void *scheduler) { memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg)); pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize; - if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -177,7 +177,7 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { exit(ret); } - if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -185,7 +185,7 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { pSched->queue[pSched->emptySlot] = *pMsg; pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize; - if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -208,21 +208,21 @@ void taosCleanUpScheduler(void *param) { } for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { - pthread_join(pSched->qthread[i], NULL); + taosThreadJoin(pSched->qthread[i], NULL); } } tsem_destroy(&pSched->emptySem); tsem_destroy(&pSched->fullSem); - pthread_mutex_destroy(&pSched->queueMutex); + taosThreadMutexDestroy(&pSched->queueMutex); if (pSched->pTimer) { taosTmrStopA(&pSched->pTimer); } - if (pSched->queue) free(pSched->queue); - if (pSched->qthread) free(pSched->qthread); - free(pSched); // fix memory leak + if (pSched->queue) taosMemoryFree(pSched->queue); + if (pSched->qthread) taosMemoryFree(pSched->qthread); + taosMemoryFree(pSched); // fix memory leak } // for debug purpose, dump the scheduler status every 1min. diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index b5e54d12d380aea8edb55f3e69a675913b6f46b2..118fe58d2e9a58b3682aeb728b63e1d4f1dd77e1 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -28,7 +28,7 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward); static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData); static SSkipListNode *tSkipListNewNode(uint8_t level); -#define tSkipListFreeNode(n) tfree((n)) +#define tSkipListFreeNode(n) taosMemoryFreeClear((n)) static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward, bool hasDup); @@ -39,7 +39,7 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, __sl_key_fn_t fn) { - SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); + SSkipList *pSkipList = (SSkipList *)taosMemoryCalloc(1, sizeof(SSkipList)); if (pSkipList == NULL) return NULL; if (maxLevel > MAX_SKIP_LIST_LEVEL) { @@ -70,13 +70,13 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ } if (SL_IS_THREAD_SAFE(pSkipList)) { - pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t)); + pSkipList->lock = (TdThreadRwlock *)taosMemoryCalloc(1, sizeof(TdThreadRwlock)); if (pSkipList->lock == NULL) { tSkipListDestroy(pSkipList); return NULL; } - if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { + if (taosThreadRwlockInit(pSkipList->lock, NULL) != 0) { tSkipListDestroy(pSkipList); return NULL; } @@ -105,17 +105,17 @@ void tSkipListDestroy(SSkipList *pSkipList) { tSkipListFreeNode(pTemp); } - tfree(pSkipList->insertHandleFn); + taosMemoryFreeClear(pSkipList->insertHandleFn); tSkipListUnlock(pSkipList); if (pSkipList->lock != NULL) { - pthread_rwlock_destroy(pSkipList->lock); - tfree(pSkipList->lock); + taosThreadRwlockDestroy(pSkipList->lock); + taosMemoryFreeClear(pSkipList->lock); } tSkipListFreeNode(pSkipList->pHead); tSkipListFreeNode(pSkipList->pTail); - tfree(pSkipList); + taosMemoryFreeClear(pSkipList); } SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { @@ -345,7 +345,7 @@ void *tSkipListDestroyIter(SSkipListIterator *iter) { return NULL; } - tfree(iter); + taosMemoryFreeClear(iter); return NULL; } @@ -418,7 +418,7 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, S } static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { - SSkipListIterator *iter = calloc(1, sizeof(SSkipListIterator)); + SSkipListIterator *iter = taosMemoryCalloc(1, sizeof(SSkipListIterator)); iter->pSkipList = pSkipList; iter->order = order; @@ -435,21 +435,21 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t static FORCE_INLINE int32_t tSkipListWLock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_wrlock(pSkipList->lock); + return taosThreadRwlockWrlock(pSkipList->lock); } return 0; } static FORCE_INLINE int32_t tSkipListRLock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_rdlock(pSkipList->lock); + return taosThreadRwlockRdlock(pSkipList->lock); } return 0; } static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_unlock(pSkipList->lock); + return taosThreadRwlockUnlock(pSkipList->lock); } return 0; } @@ -662,7 +662,7 @@ static int32_t initForwardBackwardPtr(SSkipList *pSkipList) { static SSkipListNode *tSkipListNewNode(uint8_t level) { int32_t tsize = sizeof(SSkipListNode) + sizeof(SSkipListNode *) * level * 2; - SSkipListNode *pNode = (SSkipListNode *)calloc(1, tsize); + SSkipListNode *pNode = (SSkipListNode *)taosMemoryCalloc(1, tsize); if (pNode == NULL) return NULL; pNode->level = level; diff --git a/source/util/src/tstrbuild.c b/source/util/src/tstrbuild.c index f191f69986a840c520703979f503406ab94ba9bc..2aae588046402e37569f5a2bde5ed5f72fa24346 100644 --- a/source/util/src/tstrbuild.c +++ b/source/util/src/tstrbuild.c @@ -20,7 +20,7 @@ void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size) { size += sb->pos; if (size > sb->size) { size *= 2; - void* tmp = realloc(sb->buf, size); + void* tmp = taosMemoryRealloc(sb->buf, size); if (tmp == NULL) { longjmp(sb->jb, 1); } @@ -39,7 +39,7 @@ char* taosStringBuilderGetResult(SStringBuilder* sb, size_t* len) { } void taosStringBuilderDestroy(SStringBuilder* sb) { - free(sb->buf); + taosMemoryFree(sb->buf); sb->buf = NULL; sb->pos = 0; sb->size = 0; diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index f9e28d7b6256bb61a51f8c76bc85da338c4f9d60..3a2247872972a49b6f84bae370fc4463c432c4d4 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -16,35 +16,35 @@ #define _DEFAULT_SOURCE #include "tthread.h" -pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { - pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int32_t ret = pthread_create(pthread, &thattr, __start_routine, param); - pthread_attr_destroy(&thattr); +TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param) { + TdThread* pthread = (TdThread*)taosMemoryMalloc(sizeof(TdThread)); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + int32_t ret = taosThreadCreate(pthread, &thattr, __start_routine, param); + taosThreadAttrDestroy(&thattr); if (ret != 0) { - free(pthread); + taosMemoryFree(pthread); return NULL; } return pthread; } -bool taosDestoryThread(pthread_t* pthread) { +bool taosDestoryThread(TdThread* pthread) { if (pthread == NULL) return false; if (taosThreadRunning(pthread)) { - pthread_cancel(*pthread); - pthread_join(*pthread, NULL); + taosThreadCancel(*pthread); + taosThreadJoin(*pthread, NULL); } - free(pthread); + taosMemoryFree(pthread); return true; } -bool taosThreadRunning(pthread_t* pthread) { +bool taosThreadRunning(TdThread* pthread) { if (pthread == NULL) return false; - int32_t ret = pthread_kill(*pthread, 0); + int32_t ret = taosThreadKill(*pthread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 46bca6e6cb0eaed27ebf6c0e784bddd2a33eb727..fecc58c2367c90af8c94e98e029be9874749fd2a 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -102,7 +102,7 @@ typedef struct timer_map_t { } timer_map_t; typedef struct time_wheel_t { - pthread_mutex_t mutex; + TdThreadMutex mutex; int64_t nextScanAt; uint32_t resolution; uint16_t size; @@ -112,8 +112,8 @@ typedef struct time_wheel_t { int32_t tsMaxTmrCtrl = 512; -static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t tmrCtrlMutex; +static TdThreadOnce tmrModuleInit = PTHREAD_ONCE_INIT; +static TdThreadMutex tmrCtrlMutex; static tmr_ctrl_t* tmrCtrls; static tmr_ctrl_t* unusedTmrCtrl = NULL; static void* tmrQhandle; @@ -132,7 +132,7 @@ static timer_map_t timerMap; static uintptr_t getNextTimerId() { uintptr_t id; do { - id = atomic_add_fetch_ptr(&nextTimerId, 1); + id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, 1); } while (id == 0); return id; } @@ -141,7 +141,7 @@ static void timerAddRef(tmr_obj_t* timer) { atomic_add_fetch_8(&timer->refCount, static void timerDecRef(tmr_obj_t* timer) { if (atomic_sub_fetch_8(&timer->refCount, 1) == 0) { - free(timer); + taosMemoryFree(timer); } } @@ -230,7 +230,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { timer->prev = NULL; timer->expireAt = taosGetMonotonicMs() + delay; - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); uint32_t idx = 0; if (timer->expireAt > wheel->nextScanAt) { @@ -248,7 +248,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { p->prev = timer; } - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); } static bool removeFromWheel(tmr_obj_t* timer) { @@ -259,7 +259,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { time_wheel_t* wheel = wheels + wheelIdx; bool removed = false; - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); // other thread may modify timer->wheel, check again. if (timer->wheel < tListLen(wheels)) { if (timer->prev != NULL) { @@ -277,7 +277,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { timerDecRef(timer); removed = true; } - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); return removed; } @@ -351,7 +351,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* ha return NULL; } - tmr_obj_t* timer = (tmr_obj_t*)calloc(1, sizeof(tmr_obj_t)); + tmr_obj_t* timer = (tmr_obj_t*)taosMemoryCalloc(1, sizeof(tmr_obj_t)); if (timer == NULL) { tmrError("%s failed to allocated memory for new timer object.", ctrl->label); return NULL; @@ -372,7 +372,7 @@ static void taosTimerLoopFunc(int32_t signo) { time_wheel_t* wheel = wheels + i; while (now >= wheel->nextScanAt) { - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); wheel->index = (wheel->index + 1) % wheel->size; tmr_obj_t* timer = wheel->slots[wheel->index]; while (timer != NULL) { @@ -407,7 +407,7 @@ static void taosTimerLoopFunc(int32_t signo) { timer = next; } wheel->nextScanAt += wheel->resolution; - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); } addToExpired(expired); @@ -513,7 +513,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han } static void taosTmrModuleInit(void) { - tmrCtrls = malloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl); + tmrCtrls = taosMemoryMalloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl); if (tmrCtrls == NULL) { tmrError("failed to allocate memory for timer controllers."); return; @@ -528,18 +528,18 @@ static void taosTmrModuleInit(void) { (tmrCtrls + tsMaxTmrCtrl - 1)->next = NULL; unusedTmrCtrl = tmrCtrls; - pthread_mutex_init(&tmrCtrlMutex, NULL); + taosThreadMutexInit(&tmrCtrlMutex, NULL); int64_t now = taosGetMonotonicMs(); for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; - if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { + if (taosThreadMutexInit(&wheel->mutex, NULL) != 0) { tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno)); return; } wheel->nextScanAt = now + wheel->resolution; wheel->index = 0; - wheel->slots = (tmr_obj_t**)calloc(wheel->size, sizeof(tmr_obj_t*)); + wheel->slots = (tmr_obj_t**)taosMemoryCalloc(wheel->size, sizeof(tmr_obj_t*)); if (wheel->slots == NULL) { tmrError("failed to allocate wheel slots"); return; @@ -548,7 +548,7 @@ static void taosTmrModuleInit(void) { } timerMap.count = 0; - timerMap.slots = (timer_list_t*)calloc(timerMap.size, sizeof(timer_list_t)); + timerMap.slots = (timer_list_t*)taosMemoryCalloc(timerMap.size, sizeof(timer_list_t)); if (timerMap.slots == NULL) { tmrError("failed to allocate hash map"); return; @@ -564,15 +564,15 @@ void* taosTmrInit(int32_t maxNumOfTmrs, int32_t resolution, int32_t longest, con const char* ret = taosMonotonicInit(); tmrDebug("ttimer monotonic clock source:%s", ret); - pthread_once(&tmrModuleInit, taosTmrModuleInit); + taosThreadOnce(&tmrModuleInit, taosTmrModuleInit); - pthread_mutex_lock(&tmrCtrlMutex); + taosThreadMutexLock(&tmrCtrlMutex); tmr_ctrl_t* ctrl = unusedTmrCtrl; if (ctrl != NULL) { unusedTmrCtrl = ctrl->next; numOfTmrCtrl++; } - pthread_mutex_unlock(&tmrCtrlMutex); + taosThreadMutexUnlock(&tmrCtrlMutex); if (ctrl == NULL) { tmrError("%s too many timer controllers, failed to create timer controller.", label); @@ -594,11 +594,11 @@ void taosTmrCleanUp(void* handle) { tmrDebug("%s timer controller is cleaned up.", ctrl->label); ctrl->label[0] = 0; - pthread_mutex_lock(&tmrCtrlMutex); + taosThreadMutexLock(&tmrCtrlMutex); ctrl->next = unusedTmrCtrl; numOfTmrCtrl--; unusedTmrCtrl = ctrl; - pthread_mutex_unlock(&tmrCtrlMutex); + taosThreadMutexUnlock(&tmrCtrlMutex); tmrDebug("time controller's tmr ctrl size: %d", numOfTmrCtrl); if (numOfTmrCtrl <= 0) { @@ -608,23 +608,23 @@ void taosTmrCleanUp(void* handle) { for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; - pthread_mutex_destroy(&wheel->mutex); - free(wheel->slots); + taosThreadMutexDestroy(&wheel->mutex); + taosMemoryFree(wheel->slots); } - pthread_mutex_destroy(&tmrCtrlMutex); + taosThreadMutexDestroy(&tmrCtrlMutex); for (size_t i = 0; i < timerMap.size; i++) { timer_list_t* list = timerMap.slots + i; tmr_obj_t* t = list->timers; while (t != NULL) { tmr_obj_t* next = t->mnext; - free(t); + taosMemoryFree(t); t = next; } } - free(timerMap.slots); - free(tmrCtrls); + taosMemoryFree(timerMap.slots); + taosMemoryFree(tmrCtrls); tmrCtrls = NULL; unusedTmrCtrl = NULL; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 7bd671a56ca42f9be79e123af734f1ba19ea83d6..8133e4d237f50c525ef653dd064854619e847dcc 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -147,7 +147,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { *num = 0; int32_t size = 4; - char **split = malloc(POINTER_BYTES * size); + char **split = taosMemoryMalloc(POINTER_BYTES * size); for (char *p = strsep(&z, delim); p != NULL; p = strsep(&z, delim)) { size_t len = strlen(p); @@ -158,7 +158,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { split[(*num)++] = p; if ((*num) >= size) { size = (size << 1); - split = realloc(split, POINTER_BYTES * size); + split = taosMemoryRealloc(split, POINTER_BYTES * size); assert(NULL != split); } } @@ -233,7 +233,7 @@ char *strntolower(char *dst, const char *src, int32_t n) { } } else if (c >= 'A' && c <= 'Z') { c -= 'A' - 'a'; - } else if (c == '\'' || c == '"') { + } else if (c == '\'' || c == '"' || c == '`') { quote = c; } *p++ = c; @@ -349,7 +349,7 @@ char *strbetween(char *string, char *begin, char *end) { char *_end = strstr(_begin + strlen(begin), end); int32_t size = (int32_t)(_end - _begin); if (_end != NULL && size > 0) { - result = (char *)calloc(1, size); + result = (char *)taosMemoryCalloc(1, size); memcpy(result, _begin + strlen(begin), size - +strlen(begin)); } } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 1fa70da870ffd9fe33fb24cfff9b6f3f48115154..992ec74b5b38a1009c7a3df7c66862bca810c4aa 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -22,13 +22,13 @@ typedef void *(*ThreadFp)(void *param); int32_t tQWorkerInit(SQWorkerPool *pool) { pool->qset = taosOpenQset(); - pool->workers = calloc(sizeof(SQWorker), pool->max); + pool->workers = taosMemoryCalloc(pool->max, sizeof(SQWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (pthread_mutex_init(&pool->mutex, NULL)) { + if (taosThreadMutexInit(&pool->mutex, NULL)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -56,13 +56,13 @@ void tQWorkerCleanup(SQWorkerPool *pool) { SQWorker *worker = pool->workers + i; if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { - pthread_join(worker->thread, NULL); + taosThreadJoin(worker->thread, NULL); } } - tfree(pool->workers); + taosMemoryFreeClear(pool->workers); taosCloseQset(pool->qset); - pthread_mutex_destroy(&pool->mutex); + taosThreadMutexDestroy(&pool->mutex); uDebug("worker:%s is closed", pool->name); } @@ -86,18 +86,19 @@ static void *tQWorkerThreadFp(SQWorker *worker) { } if (fp != NULL) { - (*fp)(ahandle, msg); + SQueueInfo info = {.ahandle = ahandle, .workerId = worker->id, .threadNum = pool->num}; + (*fp)(&info, msg); } } return NULL; } -STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, ThreadFp threadFp) { - pthread_mutex_lock(&pool->mutex); +STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { + taosThreadMutexLock(&pool->mutex); STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -110,11 +111,11 @@ STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, Threa do { SQWorker *worker = pool->workers + pool->num; - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, threadFp, worker) != 0) { + if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tQWorkerThreadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosCloseQueue(queue); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -122,79 +123,32 @@ STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, Threa break; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); pool->num++; uDebug("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num); } while (pool->num < pool->min); } - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); return queue; } -STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { - return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp); -} - void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } -int32_t tFWorkerInit(SFWorkerPool *pool) { return tQWorkerInit((SQWorkerPool *)pool); } - -void tFWorkerCleanup(SFWorkerPool *pool) { tQWorkerCleanup(pool); } - -static void *tFWorkerThreadFp(SQWorker *worker) { - SQWorkerPool *pool = worker->pool; - - FItem fp = NULL; - void *msg = NULL; - void *ahandle = NULL; - int32_t code = 0; - - taosBlockSIGPIPE(); - setThreadName(pool->name); - uDebug("worker:%s:%d is running", pool->name, worker->id); - - while (1) { - code = taosReadQitemFromQsetByThread(pool->qset, (void **)&msg, &ahandle, &fp, worker->id); - - if (code < 0) { - uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset); - break; - } else if (code == 0) { - // uTrace("worker:%s:%d qset:%p, got no message and continue", pool->name, worker->id, pool->qset); - continue; - } - - if (fp != NULL) { - (*fp)(ahandle, msg); - } - - taosResetQsetThread(pool->qset, msg); - } - - return NULL; -} - -STaosQueue *tFWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { - return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp); -} - -void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueue(pool, queue); } - int32_t tWWorkerInit(SWWorkerPool *pool) { pool->nextId = 0; - pool->workers = calloc(pool->max, sizeof(SWWorker)); + pool->workers = taosMemoryCalloc(pool->max, sizeof(SWWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (pthread_mutex_init(&pool->mutex, NULL) != 0) { + if (taosThreadMutexInit(&pool->mutex, NULL) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -224,14 +178,14 @@ void tWWorkerCleanup(SWWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { - pthread_join(worker->thread, NULL); + taosThreadJoin(worker->thread, NULL); taosFreeQall(worker->qall); taosCloseQset(worker->qset); } } - tfree(pool->workers); - pthread_mutex_destroy(&pool->mutex); + taosMemoryFreeClear(pool->workers); + taosThreadMutexDestroy(&pool->mutex); uInfo("worker:%s is closed", pool->name); } @@ -257,7 +211,8 @@ static void *tWWorkerThreadFp(SWWorker *worker) { } if (fp != NULL) { - (*fp)(ahandle, worker->qall, numOfMsgs); + SQueueInfo info = {.ahandle = ahandle, .workerId = worker->id, .threadNum = pool->num}; + (*fp)(&info, worker->qall, numOfMsgs); } } @@ -265,12 +220,12 @@ static void *tWWorkerThreadFp(SWWorker *worker) { } STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); SWWorker *worker = pool->workers + pool->nextId; STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -281,7 +236,7 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { worker->qset = taosOpenQset(); if (worker->qset == NULL) { taosCloseQueue(queue); - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return NULL; } @@ -290,15 +245,15 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { if (worker->qall == NULL) { taosCloseQset(worker->qset); taosCloseQueue(queue); - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { + if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosFreeQall(worker->qall); taosCloseQset(worker->qset); @@ -310,13 +265,15 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pool->nextId = (pool->nextId + 1) % pool->max; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); + pool->num++; + if (pool->num > pool->max) pool->num = pool->max; } else { taosAddIntoQset(worker->qset, queue, ahandle); pool->nextId = (pool->nextId + 1) % pool->max; } - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); return queue; @@ -326,3 +283,60 @@ void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } + +int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) { + SQWorkerPool *pPool = &pWorker->pool; + pPool->name = pCfg->name; + pPool->min = pCfg->minNum; + pPool->max = pCfg->maxNum; + if (tQWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->queue = tQWorkerAllocQueue(pPool, pCfg->param, pCfg->fp); + if (pWorker->queue == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->name = pCfg->name; + return 0; +} + +void tSingleWorkerCleanup(SSingleWorker *pWorker) { + if (pWorker->queue == NULL) return; + + while (!taosQueueEmpty(pWorker->queue)) { + taosMsleep(10); + } + + tQWorkerCleanup(&pWorker->pool); + tQWorkerFreeQueue(&pWorker->pool, pWorker->queue); +} + +int32_t tMultiWorkerInit(SMultiWorker *pWorker, const SMultiWorkerCfg *pCfg) { + SWWorkerPool *pPool = &pWorker->pool; + pPool->name = pCfg->name; + pPool->max = pCfg->maxNum; + if (tWWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->queue = tWWorkerAllocQueue(pPool, pCfg->param, pCfg->fp); + if (pWorker->queue == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->name = pCfg->name; + return 0; +} + +void tMultiWorkerCleanup(SMultiWorker *pWorker) { + if (pWorker->queue == NULL) return; + + while (!taosQueueEmpty(pWorker->queue)) { + taosMsleep(10); + } + + tWWorkerCleanup(&pWorker->pool); + tWWorkerFreeQueue(&pWorker->pool, pWorker->queue); +} diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index f4d2d9b47bf606ea2f86d1f0216a26df9445caf8..99f5a761c5d0d3a489176749883da981c847011d 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -158,7 +158,7 @@ void acquireRleaseTest() { const char *str2 = "aaaaaaa"; const char *str3 = "123456789"; - data.p = (char *)malloc(10); + data.p = (char *)taosMemoryMalloc(10); strcpy(data.p, str1); code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); @@ -177,7 +177,7 @@ void acquireRleaseTest() { strcpy(pdata->p, str3); - data.p = (char *)malloc(10); + data.p = (char *)taosMemoryMalloc(10); strcpy(data.p, str2); code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); ASSERT_EQ(code, 0); @@ -187,14 +187,14 @@ void acquireRleaseTest() { printf("%s,expect:%s", pdata->p, str3); ASSERT_TRUE(strcmp(pdata->p, str3) == 0); - tfree(pdata->p); + taosMemoryFreeClear(pdata->p); taosHashRelease(hashTable, pdata); num = taosHashGetSize(hashTable); ASSERT_EQ(num, 1); taosHashCleanup(hashTable); - tfree(data.p); + taosMemoryFreeClear(data.p); } } diff --git a/source/util/test/queueTest.cpp b/source/util/test/queueTest.cpp index 310ae4350e2f354e01fc0cced7e7506fe6df9b26..09c544df9f27f9f897d6376e5c08fbee9efe2057 100644 --- a/source/util/test/queueTest.cpp +++ b/source/util/test/queueTest.cpp @@ -14,6 +14,9 @@ #include "os.h" #include "tqueue.h" +#include +#include + class UtilTestQueue : public ::testing::Test { public: void SetUp() override {} @@ -24,6 +27,107 @@ class UtilTestQueue : public ::testing::Test { static void TearDownTestSuite() {} }; -TEST_F(UtilTestQueue, 01_ReadQitemFromQsetByThread) { - EXPECT_EQ(0, 0); -} \ No newline at end of file +#if 0 +TEST_F(UtilTestQueue, 01_fork) { + pid_t pid; + int shmid; + int* shmptr; + int* tmp; + + int err; + pthread_mutexattr_t mattr; + if ((err = pthread_mutexattr_init(&mattr)) < 0) { + printf("mutex addr init error:%s\n", strerror(err)); + exit(1); + } + + if ((err = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) < 0) { + printf("mutex addr get shared error:%s\n", strerror(err)); + exit(1); + } + + pthread_mutex_t* m; + int mid = shmget(IPC_PRIVATE, sizeof(pthread_mutex_t), 0600); + m = (pthread_mutex_t*)shmat(mid, NULL, 0); + + if ((err = pthread_mutex_init(m, &mattr)) < 0) { + printf("mutex mutex init error:%s\n", strerror(err)); + exit(1); + } + + if ((shmid = shmget(IPC_PRIVATE, 1000, IPC_CREAT | 0600)) < 0) { + perror("shmget error"); + exit(1); + } + + if ((shmptr = (int*)shmat(shmid, 0, 0)) == (void*)-1) { + perror("shmat error"); + exit(1); + } + + tmp = shmptr; + + int shmid2; + int** shmptr2; + if ((shmid2 = shmget(IPC_PRIVATE, 20, IPC_CREAT | 0600)) < 0) { + perror("shmget2 error"); + exit(1); + } + + if ((shmptr2 = (int**)shmat(shmid2, 0, 0)) == (void*)-1) { + perror("shmat2 error"); + exit(1); + } + + *shmptr2 = shmptr; + + if ((pid = fork()) < 0) { + perror("fork error"); + exit(1); + } else if (pid == 0) { + if ((err = taosThreadMutexLock(m)) < 0) { + printf("lock error:%s\n", strerror(err)); + exit(1); + } + for (int i = 0; i < 30; ++i) { + **shmptr2 = i; + (*shmptr2)++; + } + + if ((err = taosThreadMutexUnlock(m)) < 0) { + printf("unlock error:%s\n", strerror(err)); + exit(1); + } + exit(0); + + } else { + if ((err = taosThreadMutexLock(m)) < 0) { + printf("lock error:%s\n", strerror(err)); + exit(1); + } + for (int i = 10; i < 42; ++i) { + **shmptr2 = i; + (*shmptr2)++; + } + if ((err = taosThreadMutexUnlock(m)) < 0) { + printf("unlock error:%s\n", strerror(err)); + exit(1); + } + } + + wait(NULL); + + for (int i = 0; i < 70; ++i) { + printf("%d ", tmp[i]); + } + + printf("\n"); + + taosThreadAttrDestroy(&mattr); + //销毁mutex + pthread_mutex_destroy(m); + + exit(0); +} + +#endif \ No newline at end of file diff --git a/source/util/test/skiplistTest.cpp b/source/util/test/skiplistTest.cpp index 0b629f64aab9e07851af2d7b96e80121f4aaa2a4..7110b21aa9f6339c43a216a9c3c42393628d99b3 100644 --- a/source/util/test/skiplistTest.cpp +++ b/source/util/test/skiplistTest.cpp @@ -32,7 +32,7 @@ void doubleSkipListTest() { size = 0; // tSkipListNewNodeInfo(pSkipList, &level, &size); - // auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); + // auto d = (SSkipListNode*)taosMemoryCalloc(1, size + sizeof(double) * 2); // d->level = level; double key = 0.997; @@ -59,7 +59,7 @@ void doubleSkipListTest() { } if (size > 0) { - tfree(pNodes); + taosMemoryFreeClear(pNodes); } } @@ -83,7 +83,7 @@ void randKeyTest() { int32_t s = 0; tSkipListNewNodeInfo(pSkipList, &level, &s); - auto d = (SSkipListNode*)calloc(1, s + sizeof(int32_t) * 2); + auto d = (SSkipListNode*)taosMemoryCalloc(1, s + sizeof(int32_t) * 2); d->level = level; int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); @@ -115,7 +115,7 @@ void stringKeySkiplistTest() { int32_t headsize = 0; tSkipListNewNodeInfo(pSkipList, &level, &headsize); - auto pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + auto pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + max_key_size + sizeof(double)); pNode->level = level; char* d = SL_GET_NODE_DATA(pNode); @@ -127,7 +127,7 @@ void stringKeySkiplistTest() { tSkipListNewNodeInfo(pSkipList, &level, &headsize); - pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + max_key_size + sizeof(double)); pNode->level = level; d = SL_GET_NODE_DATA(pNode); @@ -153,7 +153,7 @@ void stringKeySkiplistTest() { tSkipListDestroy(pSkipList); - free(pRes); + taosMemoryFree(pRes); #endif tSkipListDestroy(pSkipList); @@ -167,7 +167,7 @@ void stringKeySkiplistTest() { int32_t n = sprintf(k, "abc_%d_%d", i, i); tSkipListNewNodeInfo(pSkipList, &level, &headsize); - auto pNode = (SSkipListNode*)calloc(1, headsize + 20 + sizeof(double)); + auto pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + 20 + sizeof(double)); pNode->level = level; char* d = SL_GET_NODE_DATA(pNode); @@ -197,7 +197,7 @@ void stringKeySkiplistTest() { tSkipListRemoveNode(pSkipList, pres[0]); if (num > 0) { - tfree(pres); + taosMemoryFreeClear(pres); } } @@ -219,7 +219,7 @@ void skiplistPerformanceTest() { int32_t unit = MAX_SKIP_LIST_LEVEL * POINTER_BYTES * 2 + sizeof(double) * 2 + sizeof(int16_t); - char* total = (char*)calloc(1, unit * size); + char* total = (char*)taosMemoryCalloc(1, unit * size); char* p = total; for (int32_t i = 0; i < size; ++i) { @@ -277,7 +277,7 @@ void skiplistPerformanceTest() { assert(SL_GET_SIZE(pSkipList) == size); tSkipListDestroy(pSkipList); - tfree(total); + taosMemoryFreeClear(total); } // todo not support duplicated key yet @@ -288,7 +288,7 @@ void duplicatedKeyTest() { for (int32_t j = 0; j < 5; ++j) { int32_t level, size; tSkipListNewNodeInfo(pSkipList, &level, &size); - SSkipListNode* d = (SSkipListNode*)calloc(1, size + sizeof(int32_t)); + SSkipListNode* d = (SSkipListNode*)taosMemoryCalloc(1, size + sizeof(int32_t)); d->level = level; int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); key[0] = i; @@ -358,9 +358,9 @@ TEST(testCase, skiplist_test) { printf("-----%lf\n", pNodes[i]->key.dKey); } printf("the range query result size is: %d\n", size); - tfree(pNodes); + taosMemoryFreeClear(pNodes); - SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); + SSkipListKey *pKeys = taosMemoryMalloc(sizeof(SSkipListKey) * 20); for (int32_t i = 0; i < 8; i += 2) { pKeys[i].dKey = i * 0.997; pKeys[i].nType = TSDB_DATA_TYPE_DOUBLE; @@ -372,9 +372,9 @@ TEST(testCase, skiplist_test) { for (int32_t i = 0; i < r; ++i) { // printf("%lf ", pNodes[i]->key.dKey); } - tfree(pNodes); + taosMemoryFreeClear(pNodes); - free(pKeys);*/ + taosMemoryFree(pKeys);*/ } #endif \ No newline at end of file diff --git a/source/util/test/stringTest.cpp b/source/util/test/stringTest.cpp index 95fba0cd3e2b62da2b2dadb0b7e592aef5893543..dee5fb30d30661d38413b2610a331b7c19bddd15 100644 --- a/source/util/test/stringTest.cpp +++ b/source/util/test/stringTest.cpp @@ -33,35 +33,35 @@ TEST(testCase, string_replace_test) { EXPECT_EQ(strlen(ret), 7); EXPECT_STREQ("7017027", ret); - free(ret); + taosMemoryFree(ret); char t4[] = "a01a02b03c04d05"; ret = strreplace(t4, "0", "9999999999"); EXPECT_EQ(strlen(ret), 5 * 10 + 10); EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret); - free(ret); + taosMemoryFree(ret); char t5[] = "abc"; ret = strreplace(t5, "abc", "12345678901234567890"); EXPECT_EQ(strlen(ret), 20); EXPECT_STREQ("12345678901234567890", ret); - free(ret); + taosMemoryFree(ret); char t6[] = "abc"; ret = strreplace(t6, "def", "abc"); EXPECT_EQ(strlen(ret), 3); EXPECT_STREQ("abc", ret); - free(ret); + taosMemoryFree(ret); char t7[] = "abcde000000000000001234"; ret = strreplace(t7, "ab", "0000000"); EXPECT_EQ(strlen(ret), 28); EXPECT_STREQ("0000000cde000000000000001234", ret); - free(ret); + taosMemoryFree(ret); char t8[] = "abc\ndef"; char t[] = {10, 0}; @@ -72,21 +72,21 @@ TEST(testCase, string_replace_test) { EXPECT_EQ(strlen(ret), 8); EXPECT_STREQ("abc\\ndef", ret); - free(ret); + taosMemoryFree(ret); char t9[] = "abc\\ndef"; ret = strreplace(t9, "\\n", "\n"); EXPECT_EQ(strlen(ret), 7); EXPECT_STREQ("abc\ndef", ret); - free(ret); + taosMemoryFree(ret); char t10[] = "abcdef"; ret = strreplace(t10, "", "0"); EXPECT_EQ(strlen(ret), 6); EXPECT_STREQ("abcdef", ret); - free(ret); + taosMemoryFree(ret); } #endif diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 58d9d2202ec66459e50799271332e1d398a72ad6..3174d57aef9b50583fc0b28d7bf389572bebdd0c 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -1,6 +1,5 @@ #include #include -#include #include #include #include "os.h" @@ -39,7 +38,7 @@ void *addRef(void *param) { printf("a"); id = random() % pSpace->refNum; if (pSpace->rid[id] <= 0) { - pSpace->p[id] = malloc(128); + pSpace->p[id] = taosMemoryMalloc(128); pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); } taosUsleep(100); @@ -85,7 +84,7 @@ void *acquireRelease(void *param) { } void myfree(void *p) { - free(p); + taosMemoryFree(p); } void *openRefSpace(void *param) { @@ -99,21 +98,21 @@ void *openRefSpace(void *param) { return NULL; } - pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); - pSpace->rid = calloc(pSpace->refNum, sizeof(int64_t)); + pSpace->p = (void **) taosMemoryCalloc(sizeof(void *), pSpace->refNum); + pSpace->rid = taosMemoryCalloc(pSpace->refNum, sizeof(int64_t)); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_t thread1, thread2, thread3; - pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace)); - pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace)); - pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace)); + TdThread thread1, thread2, thread3; + taosThreadCreate(&(thread1), &thattr, addRef, (void *)(pSpace)); + taosThreadCreate(&(thread2), &thattr, removeRef, (void *)(pSpace)); + taosThreadCreate(&(thread3), &thattr, acquireRelease, (void *)(pSpace)); - pthread_join(thread1, NULL); - pthread_join(thread2, NULL); - pthread_join(thread3, NULL); + taosThreadJoin(thread1, NULL); + taosThreadJoin(thread2, NULL); + taosThreadJoin(thread3, NULL); for (int i=0; irefNum; ++i) { taosRemoveRef(pSpace->rsetId, pSpace->rid[i]); @@ -122,7 +121,7 @@ void *openRefSpace(void *param) { taosCloseRef(pSpace->rsetId); uInfo("rsetId:%d main thread exit", pSpace->rsetId); - free(pSpace->p); + taosMemoryFree(pSpace->p); pSpace->p = NULL; return NULL; @@ -160,31 +159,31 @@ int main(int argc, char *argv[]) { taosInitLog("tref.log", 10); - SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads); + SRefSpace *pSpaceList = (SRefSpace *) taosMemoryCalloc(sizeof(SRefSpace), threads); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), threads); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i=0; i #include #include #include @@ -228,13 +227,13 @@ int stmt_scol_func3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -333,11 +332,11 @@ int stmt_scol_func3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -358,13 +357,13 @@ int stmt_scol_func4(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -456,11 +455,11 @@ int stmt_scol_func4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -855,18 +854,18 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1041,12 +1040,12 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1068,18 +1067,18 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1254,12 +1253,12 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1282,18 +1281,18 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1443,12 +1442,12 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1473,18 +1472,18 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1598,12 +1597,12 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1626,18 +1625,18 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1787,12 +1786,12 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1815,18 +1814,18 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2002,12 +2001,12 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2032,18 +2031,18 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2220,12 +2219,12 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2247,18 +2246,18 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2445,12 +2444,12 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2474,18 +2473,18 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2672,12 +2671,12 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2698,13 +2697,13 @@ int stmt_funcb1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -2831,11 +2830,11 @@ int stmt_funcb1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2855,13 +2854,13 @@ int stmt_funcb2(TAOS_STMT *stmt) { char bin[18000][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(18000 * sizeof(int)); + int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = malloc(sizeof(char) * 18000); - char* no_null = malloc(sizeof(char) * 18000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 18000); + char* no_null = taosMemoryMalloc(sizeof(char) * 18000); for (int i = 0; i < 18000; ++i) { lb[i] = 40; @@ -2989,11 +2988,11 @@ int stmt_funcb2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3013,13 +3012,13 @@ int stmt_funcb3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3152,11 +3151,11 @@ int stmt_funcb3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3178,13 +3177,13 @@ int stmt_funcb4(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3311,11 +3310,11 @@ int stmt_funcb4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3337,13 +3336,13 @@ int stmt_funcb5(TAOS_STMT *stmt) { char bin[18000][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(18000 * sizeof(int)); + int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = malloc(sizeof(char) * 18000); - char* no_null = malloc(sizeof(char) * 18000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 18000); + char* no_null = taosMemoryMalloc(sizeof(char) * 18000); for (int i = 0; i < 18000; ++i) { lb[i] = 40; @@ -3464,11 +3463,11 @@ int stmt_funcb5(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3481,12 +3480,12 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { int b[30000]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 30000 * 3000); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 30000 * 3000); - int *lb = malloc(30000 * sizeof(int)); + int *lb = taosMemoryMalloc(30000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* no_null = malloc(sizeof(int) * 200000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* no_null = taosMemoryMalloc(sizeof(int) * 200000); for (int i = 0; i < 30000; ++i) { lb[i] = 40; @@ -3549,10 +3548,10 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(no_null); return 0; } @@ -3572,13 +3571,13 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3706,11 +3705,11 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3734,13 +3733,13 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3868,11 +3867,11 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3892,13 +3891,13 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -4028,11 +4027,11 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4052,13 +4051,13 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -4185,11 +4184,11 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4248,7 +4247,7 @@ int sql_perf1(TAOS *taos) { TAOS_RES *result; for (int i = 0; i < 3000; i++) { - sql[i] = calloc(1, 1048576); + sql[i] = taosMemoryCalloc(1, 1048576); } int len = 0; @@ -4280,7 +4279,7 @@ int sql_perf1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } return 0; @@ -4292,11 +4291,11 @@ int sql_perf1(TAOS *taos) { //one table 60 records one time int sql_perf_s1(TAOS *taos) { - char **sql = calloc(1, sizeof(char*) * 360000); + char **sql = taosMemoryCalloc(1, sizeof(char*) * 360000); TAOS_RES *result; for (int i = 0; i < 360000; i++) { - sql[i] = calloc(1, 9000); + sql[i] = taosMemoryCalloc(1, 9000); } int len = 0; @@ -4333,10 +4332,10 @@ int sql_perf_s1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 360000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } - free(sql); + taosMemoryFree(sql); return 0; } @@ -4348,7 +4347,7 @@ int sql_s_perf1(TAOS *taos) { TAOS_RES *result; for (int i = 0; i < 3000; i++) { - sql[i] = calloc(1, 1048576); + sql[i] = taosMemoryCalloc(1, 1048576); } int len = 0; @@ -4380,7 +4379,7 @@ int sql_s_perf1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } return 0; @@ -5067,11 +5066,11 @@ int main(int argc, char *argv[]) exit(1); } - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), 4); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), 4); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); T_par par[4]; par[0].taos = taos[0]; @@ -5083,10 +5082,10 @@ int main(int argc, char *argv[]) par[3].taos = taos[3]; par[3].idx = 3; - pthread_create(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); - //pthread_create(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); - //pthread_create(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); - //pthread_create(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); + taosThreadCreate(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); + //taosThreadCreate(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); + //taosThreadCreate(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); + //taosThreadCreate(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); while(1) { taosSsleep(1); diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index 57cbdf10900c36f9d4ce155caad4d64cd93b0078..b671b8bc068f0748e325714d12a23c4efa78e336 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -1,7 +1,6 @@ // TAOS standard API example. The same syntax as MySQL, but only a subet // to compile: gcc -o prepare prepare.c -ltaos -#include #include #include #include @@ -51,19 +50,19 @@ unsigned long long getCurrentTime(){ } static int stmt_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -222,33 +221,33 @@ static int stmt_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -407,33 +406,33 @@ static int stmt_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -529,32 +528,32 @@ static int stmt_bind_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_004(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -731,32 +730,32 @@ static int stmt_bind_case_004(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -926,33 +925,33 @@ static int stmt_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -1111,32 +1110,32 @@ static int stmt_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -1315,14 +1314,14 @@ static int stmt_bind_error_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2059,19 +2058,19 @@ static void runCase(TAOS *taos) { static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum, int64_t* startTs) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = *startTs; @@ -2232,14 +2231,14 @@ static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerC unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2326,19 +2325,19 @@ static void runCase_long(TAOS *taos) { test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2456,14 +2455,14 @@ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rows unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2473,19 +2472,19 @@ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rows test scene: insert into ? (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2603,14 +2602,14 @@ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rows unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2620,19 +2619,19 @@ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rows test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2729,14 +2728,14 @@ static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3206,20 +3205,20 @@ static void SpecifyColumnBatchCase(TAOS *taos) { test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3388,34 +3387,34 @@ static int stmt_specifyCol_bind_case_001_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_specifyCol_bind_case_002_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3584,35 +3583,35 @@ static int stmt_specifyCol_bind_case_002_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } // some tags are null static int stmt_specifyCol_bind_case_003_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int one_is_null = 1; @@ -3782,35 +3781,35 @@ static int stmt_specifyCol_bind_case_003_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } // specify tags field, and not support , then is error case static int stmt_specifyCol_bind_case_004_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3979,15 +3978,15 @@ static int stmt_specifyCol_bind_case_004_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4414,19 +4413,19 @@ char * taos_stmt_errstr(TAOS_STMT *stmt) 用于在其他stmt API 返回错误（ 3. 返回的错误码对于的错误消息； */ static int stmt_specifyCol_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -4571,32 +4570,32 @@ static int stmt_specifyCol_bind_error_case_001(TAOS_STMT *stmt, int tableNum, in unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_specifyCol_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -4716,14 +4715,14 @@ static int stmt_specifyCol_bind_error_case_002(TAOS_STMT *stmt, int tableNum, in unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -5080,8 +5079,8 @@ int main(int argc, char *argv[]) #if 0 printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), (size_t)threadNum); - ThreadInfo* threadInfo = (ThreadInfo *) calloc(sizeof(ThreadInfo), (size_t)threadNum); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), (size_t)threadNum); + ThreadInfo* threadInfo = (ThreadInfo *) taosMemoryCalloc(sizeof(ThreadInfo), (size_t)threadNum); ThreadInfo* tInfo = threadInfo; for (int i = 0; i < threadNum; i++) { @@ -5094,20 +5093,20 @@ int main(int argc, char *argv[]) tInfo->taos = taos; tInfo->idx = i; if (0 == i) { - //pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); - pthread_create(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); + //taosThreadCreate(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + taosThreadCreate(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); } else if (1 == i){ - pthread_create(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); + taosThreadCreate(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); } tInfo++; } for (int i = 0; i < threadNum; i++) { - pthread_join(pThreadList[i], NULL); + taosThreadJoin(pThreadList[i], NULL); } - free(pThreadList); - free(threadInfo); + taosMemoryFree(pThreadList); + taosMemoryFree(threadInfo); #endif taos = taos_connect(serverIp, "root", "taosdata", NULL, 0); diff --git a/tests/script/api/stmtTest.c b/tests/script/api/stmtTest.c index c69de46f8456b50e575db6b55e9bc9a7fa179543..02d65de5677c96cea5256115bfb9a899dabe5f01 100644 --- a/tests/script/api/stmtTest.c +++ b/tests/script/api/stmtTest.c @@ -1,4 +1,3 @@ -#include #include #include #include @@ -64,7 +63,7 @@ int main(int argc, char *argv[]) { execute_simple_sql(taos, "use test"); execute_simple_sql(taos, "create table super(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 tinyint, c8 bool, c9 nchar(8), c10 timestamp) tags (t1 int, t2 bigint, t3 float, t4 double, t5 binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8))"); - char *sql = calloc(1, 1024*1024); + char *sql = taosMemoryCalloc(1, 1024*1024); int sqlLen = 0; sqlLen = sprintf(sql, "create table"); for (int i = 0; i < 10; i++) { diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c index 4521ad4fa300e1ad9caca8cbaddbbbfd376e0045..62f8fab29f1cf964bc9c9813a74191e080908e3e 100644 --- a/tests/script/api/stmt_function.c +++ b/tests/script/api/stmt_function.c @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -51,7 +50,7 @@ void taos_stmt_init_test() { } void taos_stmt_preprare_test() { printf("start taos_stmt_prepare test\n"); - char *stmt_sql = calloc(1, 1048576); + char *stmt_sql = taosMemoryCalloc(1, 1048576); TAOS_STMT *stmt = NULL; assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); void *taos = NULL; @@ -94,14 +93,14 @@ void taos_stmt_preprare_test() { assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_close(stmt) == 0); - free(stmt_sql); + taosMemoryFree(stmt_sql); printf("finish taos_stmt_prepare test\n"); } void taos_stmt_set_tbname_test() { printf("start taos_stmt_set_tbname test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1, 200); + char *name = taosMemoryCalloc(1, 200); // ASM ERROR // assert(taos_stmt_set_tbname(stmt, name) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -116,14 +115,14 @@ void taos_stmt_set_tbname_test() { stmt = taos_stmt_init(taos); assert(stmt != NULL); assert(taos_stmt_set_tbname(stmt, name) != 0); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); sprintf(name, "super"); assert(stmt != NULL); assert(taos_stmt_set_tbname(stmt, name) == 0); - free(name); - free(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(stmt_sql); taos_stmt_close(stmt); printf("finish taos_stmt_set_tbname test\n"); } @@ -131,8 +130,8 @@ void taos_stmt_set_tbname_test() { void taos_stmt_set_tbname_tags_test() { printf("start taos_stmt_set_tbname_tags test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1,20); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND)); + char *name = taosMemoryCalloc(1,20); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND)); // ASM ERROR // assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -147,7 +146,7 @@ void taos_stmt_set_tbname_tags_test() { execute_simple_sql(taos, "create table tb using super tags (1)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? using super tags (?) values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); @@ -160,9 +159,9 @@ void taos_stmt_set_tbname_tags_test() { tags->length = &tags->buffer_length; tags->is_null = NULL; assert(taos_stmt_set_tbname_tags(stmt, name, tags) == 0); - free(stmt_sql); - free(name); - free(tags); + taosMemoryFree(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(tags); taos_stmt_close(stmt); printf("finish taos_stmt_set_tbname_tags test\n"); } @@ -170,7 +169,7 @@ void taos_stmt_set_tbname_tags_test() { void taos_stmt_set_sub_tbname_test() { printf("start taos_stmt_set_sub_tbname test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1, 200); + char *name = taosMemoryCalloc(1, 200); // ASM ERROR // assert(taos_stmt_set_sub_tbname(stmt, name) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -185,7 +184,7 @@ void taos_stmt_set_sub_tbname_test() { execute_simple_sql(taos, "create table tb using super tags (1)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_set_sub_tbname(stmt, name) != 0); @@ -193,8 +192,8 @@ void taos_stmt_set_sub_tbname_test() { assert(taos_stmt_set_sub_tbname(stmt, name) == 0); // assert(taos_load_table_info(taos, "super, tb") == 0); // assert(taos_stmt_set_sub_tbname(stmt, name) == 0); - free(name); - free(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_set_sub_tbname test\n"); } @@ -214,12 +213,12 @@ void taos_stmt_bind_param_test() { execute_simple_sql(taos, "use stmt_test"); execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); stmt = taos_stmt_init(taos); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_bind_param(stmt, binds) != 0); - free(binds); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + taosMemoryFree(binds); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -235,8 +234,8 @@ void taos_stmt_bind_param_test() { assert(taos_stmt_bind_param(stmt, params) != 0); assert(taos_stmt_set_tbname(stmt, "super") == 0); assert(taos_stmt_bind_param(stmt, params) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); taos_stmt_close(stmt); printf("finish taos_stmt_bind_param test\n"); } @@ -272,11 +271,11 @@ void taos_stmt_add_batch_test() { execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_add_batch(stmt) != 0); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -292,8 +291,8 @@ void taos_stmt_add_batch_test() { assert(taos_stmt_set_tbname(stmt, "super") == 0); assert(taos_stmt_bind_param(stmt, params) == 0); assert(taos_stmt_add_batch(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_add_batch test\n"); } @@ -314,11 +313,11 @@ void taos_stmt_execute_test() { stmt = taos_stmt_init(taos); assert(stmt != NULL); assert(taos_stmt_execute(stmt) != 0); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_execute(stmt) != 0); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -337,8 +336,8 @@ void taos_stmt_execute_test() { assert(taos_stmt_execute(stmt) != 0); assert(taos_stmt_add_batch(stmt) == 0); assert(taos_stmt_execute(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_execute test\n"); } @@ -346,7 +345,7 @@ void taos_stmt_execute_test() { void taos_stmt_use_result_query(void *taos, char *col, int type) { TAOS_STMT *stmt = taos_stmt_init(taos); assert(stmt != NULL); - char *stmt_sql = calloc(1, 1024); + char *stmt_sql = taosMemoryCalloc(1, 1024); struct { int64_t c1; int32_t c2; @@ -373,7 +372,7 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { sprintf(stmt_sql, "select * from stmt_test.t1 where %s = ?", col); printf("stmt_sql: %s\n", stmt_sql); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); - TAOS_BIND *params = calloc(1, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_BIND)); params->buffer_type = type; params->is_null = NULL; switch(type){ @@ -435,8 +434,8 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { assert(result != NULL); print_result(result); assert(taos_stmt_close(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); taos_free_result(result); } diff --git a/tests/script/http/httpTest.c b/tests/script/http/httpTest.c index 36ce6b95ba7836893aeec0831d60a289ff658f66..0298d5507dd7f54c3a5aa4ac95961f6a82f7c428 100644 --- a/tests/script/http/httpTest.c +++ b/tests/script/http/httpTest.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -78,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sql"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sql/db%d",pThread->threadId); @@ -95,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -104,21 +103,21 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/http/httpTestSqlUtc.c b/tests/script/http/httpTestSqlUtc.c index 643c884a1a64d6eaaeb8984cd80d985408edf0e6..b93f13462395f5ed9035caf1f42969e6288964e1 100644 --- a/tests/script/http/httpTestSqlUtc.c +++ b/tests/script/http/httpTestSqlUtc.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -78,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sqlutc"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sqlutc/db%d",pThread->threadId); @@ -95,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -104,21 +103,21 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/http/httpTestSqlt.c b/tests/script/http/httpTestSqlt.c index 2eaaee0f992d802d57b4fc4d684da4622ea3b763..d8d21ac137f836b2d83a5997c2eab3479416106a 100644 --- a/tests/script/http/httpTestSqlt.c +++ b/tests/script/http/httpTestSqlt.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -78,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sqlt"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sqlt/db%d",pThread->threadId); @@ -95,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -104,21 +103,21 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0c2b4fe1f4dad86661e70b06282b3cd8e9db86a9..dd44baed27734b3574091b1f29d7d1c63e68203b 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -10,15 +10,26 @@ ./test.sh -f tsim/db/basic7.sim ./test.sh -f tsim/db/error1.sim -# ---- table -./test.sh -f tsim/table/basic1.sim - # ---- dnode ./test.sh -f tsim/dnode/basic1.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim +./test.sh -f tsim/insert/basic1.sim +./test.sh -f tsim/insert/backquote.sim +./test.sh -f tsim/insert/null.sim # ---- query ./test.sh -f tsim/query/interval.sim +./test.sh -f tsim/query/interval-offset.sim + +# ---- show +./test.sh -f tsim/show/basic.sim + +# ---- table +./test.sh -f tsim/table/basic1.sim + +# ---- tmq +./test.sh -f tsim/tmq/basic.sim + #======================b1-end=============== diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 05f756ebb6d01b82d55692a9f9aa06585592990b..8d7dba2de2a456a21e051ffa04acecf17157af25 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -74,7 +74,7 @@ BUILD_DIR=$TAOS_DIR/$BIN_DIR SIM_DIR=$TAOS_DIR/sim NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/source/dnode/mgmt/daemon +EXE_DIR=$BUILD_DIR/source/dnode/mgmt/main CFG_DIR=$NODE_DIR/cfg LOG_DIR=$NODE_DIR/log DATA_DIR=$NODE_DIR/data diff --git a/tests/script/sh/massiveTable/compileVersion.sh b/tests/script/sh/massiveTable/compileVersion.sh index c6c92bf72458c110ffa25a0127468d0bf2723c99..dd6382992ac8cfa08a6ec01a921059da61e0683f 100755 --- a/tests/script/sh/massiveTable/compileVersion.sh +++ b/tests/script/sh/massiveTable/compileVersion.sh @@ -68,7 +68,7 @@ gitPullBranchInfo $TDengineBrVer compileTDengineVersion taos_dir=${projectDir}/debug/tools/shell -taosd_dir=${projectDir}/debug/source/dnode/mgmt/daemon +taosd_dir=${projectDir}/debug/source/dnode/mgmt/main exec_process_dir=${projectDir}/debug/tests/test/c rm -f /usr/bin/taos diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c index b4b0b81a0677b7de102dcfd228d09e53e3edda62..02979203617cf9da2b8e5b18bf326b56a45734d7 100644 --- a/tests/script/sh/sum_double.c +++ b/tests/script/sh/sum_double.c @@ -71,14 +71,14 @@ void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* int sum_double_init(SUdfInit* buf) { buf->maybe_null=1; - buf->ptr = malloc(sizeof(int)); + buf->ptr = taosMemoryMalloc(sizeof(int)); printf("sum_double init\n"); return 0; } void sum_double_destroy(SUdfInit* buf) { - free(buf->ptr); + taosMemoryFree(buf->ptr); printf("sum_double destroy\n"); } diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim index 21e26784bf0ff9399536be98b3ca6ee3a3993c27..c07ebd040002b864b6533f42d929ed898b890331 100644 --- a/tests/script/tsim/db/basic1.sim +++ b/tests/script/tsim/db/basic1.sim @@ -6,7 +6,7 @@ sql connect print =============== create database sql create database d1 vgroups 2 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -40,7 +40,7 @@ endi print =============== drop database sql drop database d1 sql show databases -if $rows != 0 then +if $rows != 1 then return -1 endi @@ -49,7 +49,7 @@ sql create database d2 vgroups 2 sql create database d3 vgroups 3 sql create database d4 vgroups 4 sql show databases -if $rows != 3 then +if $rows != 4 then return -1 endi @@ -111,7 +111,7 @@ print =============== drop database sql drop database d2 sql drop database d3 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -154,7 +154,7 @@ system sh/exec.sh -n dnode1 -s start print =============== show databases sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 08ce9955b8d62e5d8cd422de2b6c8ddacbe8c2bc..7e57fe8f1b3458e70772d50c83a60ac4831855c6 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -20,7 +20,7 @@ sql show databases print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $rows != 1 then +if $rows != 2 then return -1 endi if $data00 != $db then @@ -52,14 +52,14 @@ print =============== step2 sql_error create database $db sql create database if not exists $db sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi print =============== step3 sql drop database $db sql show databases -if $rows != 0 then +if $rows != 1 then return -1 endi @@ -318,4 +318,4 @@ if $rows != 0 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/basic7.sim b/tests/script/tsim/db/basic7.sim index 27263ac606468bc188643d032c160fc80fce9d08..73651eed2f5dbf06c35d4b553d963bd610f6c97a 100644 --- a/tests/script/tsim/db/basic7.sim +++ b/tests/script/tsim/db/basic7.sim @@ -14,6 +14,7 @@ sql insert into tb1 values (now, 1); sql show stables if $rows != 1 then + print $rows return -1 endi diff --git a/tests/script/tsim/db/error1.sim b/tests/script/tsim/db/error1.sim index bf9e04c017b79fc436a4a7d11547ec8071a183c6..6f62228ae73d5dc34f908ed969c0f6ff15d62392 100644 --- a/tests/script/tsim/db/error1.sim +++ b/tests/script/tsim/db/error1.sim @@ -24,8 +24,8 @@ endi print ========== stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGKILL -print =============== create database -sql_error create database d1 vgroups 4 +#print =============== create database +#sql_error create database d1 vgroups 4 print ========== start dnode2 system sh/exec.sh -n dnode2 -s start @@ -42,7 +42,7 @@ re-create1: sql create database d1 vgroups 2 -x re-create1 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -61,11 +61,13 @@ endi print ========== stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGKILL +sleep 1000 print =============== create database sql_error drop database d1 print ========== start dnode2 system sh/exec.sh -n dnode2 -s start +sleep 1000 print =============== re-create database $x = 0 @@ -79,7 +81,7 @@ re-create2: sql create database d1 vgroups 5 -x re-create2 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -96,4 +98,4 @@ if $data03 != 0 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim index 33e62de519150fca92440ba11c17ac0d35ae9bf1..6f0d5f88b812c52f2e355ab51c63f9950df1d653 100644 --- a/tests/script/tsim/dnode/basic1.sim +++ b/tests/script/tsim/dnode/basic1.sim @@ -15,9 +15,10 @@ if $data00 != 1 then return -1 endi -if $data02 != 0 then - return -1 -endi +# check 'vnodes' feild ? +#if $data02 != 0 then +# return -1 +#endi sql show mnodes; if $rows != 1 then @@ -49,9 +50,10 @@ if $data10 != 2 then return -1 endi -if $data02 != 0 then - return -1 -endi +# check 'vnodes' feild ? +#if $data02 != 0 then +# return -1 +#endi if $data12 != 0 then return -1 @@ -83,7 +85,7 @@ sql create database d1 vgroups 4; sql create database d2; sql show databases -if $rows != 2 then +if $rows != 3 then return -1 endi @@ -194,4 +196,4 @@ if $data00 != 1 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/insert/backquote.sim b/tests/script/tsim/insert/backquote.sim new file mode 100644 index 0000000000000000000000000000000000000000..0c8b3df4afd20b85c9d44c743246fb60031fd295 --- /dev/null +++ b/tests/script/tsim/insert/backquote.sim @@ -0,0 +1,353 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database `database` +sql create database `DataBase` +sql show databases +print rows: $rows +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +if $rows != 3 then + return -1 +endi +if $data00 != database then + return -1 +endi +if $data10 != DataBase then + return -1 +endi +if $data20 != information_schema then + return -1 +endi + +$dbCnt = 0 +while $dbCnt < 2 + if $dbCnt == 0 then + sql use `database` + else + sql use `DataBase` + endi + + $dbCnt = $dbCnt + 1 + + print =============== create super table, include all type + print notes: after nchar show ok, modify binary to nchar + sql create table `stable` (`timestamp` timestamp, `int` int, `binary` binary(16), `nchar` nchar(16)) tags (`float` float, `Binary` binary(16), `Nchar` nchar(16)) + sql create table `Stable` (`timestamp` timestamp, `int` int, `Binary` binary(32), `Nchar` nchar(32)) tags (`float` float, `binary` binary(16), `nchar` nchar(16)) + + sql show stables + print rows: $rows + print $data00 $data01 + print $data10 $data11 + if $rows != 2 then + return -1 + endi + if $data00 != Stable then + if $data00 != stable then + return -1 + endi + endi + if $data10 != Stable then + if $data10 != stable then + return -1 + endi + endi + + print =============== create child table + sql create table `table` using `stable` tags(100.0, 'stable+table', 'stable+table') + sql create table `Table` using `stable` tags(100.1, 'stable+Table', 'stable+Table') + + sql create table `TAble` using `Stable` tags(100.0, 'Stable+TAble', 'Stable+TAble') + sql create table `TABle` using `Stable` tags(100.1, 'Stable+TABle', 'Stable+TABle') + + sql show tables + print rows: $rows + print $data00 $data01 + print $data10 $data11 + print $data20 $data21 + print $data30 $data31 + if $rows != 4 then + return -1 + endi + + print =============== insert data + sql insert into `table` values(now+0s, 10, 'table', 'table')(now+1s, 11, 'table', 'table') + sql insert into `Table` values(now+0s, 20, 'Table', 'Table')(now+1s, 21, 'Table', 'Table') + sql insert into `TAble` values(now+0s, 30, 'TAble', 'TAble')(now+1s, 31, 'TAble', 'TAble') + sql insert into `TABle` values(now+0s, 40, 'TABle', 'TABle')(now+4s, 41, 'TABle', 'TABle') + + print =============== query data + sql select * from `table` + print rows: $rows + print $data00 $data01 $data02 $data03 + print $data10 $data11 $data12 $data13 + if $rows != 2 then + return -1 + endi + if $data01 != 10 then + return -1 + endi + if $data02 != table then + return -1 + endi + if $data03 != table then + print expect table, actual $data03 + return -1 + endi + + sql select * from `Table` + print rows: $rows + print $data00 $data01 $data02 $data03 + print $data10 $data11 $data12 $data13 + if $rows != 2 then + return -1 + endi + if $data01 != 20 then + return -1 + endi + if $data02 != Table then + return -1 + endi + if $data03 != Table then + return -1 + endi + + sql select * from `TAble` + print rows: $rows + print $data00 $data01 $data02 $data03 + print $data10 $data11 $data12 $data13 + if $rows != 2 then + return -1 + endi + if $data01 != 30 then + return -1 + endi + if $data02 != TAble then + return -1 + endi + if $data03 != TAble then + return -1 + endi + + sql select * from `TABle` + print rows: $rows + print $data00 $data01 $data02 $data03 + print $data10 $data11 $data12 $data13 + if $rows != 2 then + return -1 + endi + if $data01 != 40 then + return -1 + endi + if $data02 != TABle then + return -1 + endi + if $data03 != TABle then + return -1 + endi + + #print =============== query data from st, but not support select * from super table, waiting fix + #sql select count(*) from `stable` + #print rows: $rows + #print $data00 $data01 $data02 $data03 + #if $rows != 1 then + # return -1 + #endi + #if $data00 != 4 then + # return -1 + #endi + #sql select count(*) from `Stable` + #print rows: $rows + #print $data00 $data01 $data02 $data03 + #if $rows != 1 then + # return -1 + #endi + #if $data00 != 4 then + # return -1 + #endi + #sql select * from `stable` + #if $rows != 4 then + # return -1 + #endi + +endw + +print =============== stop and restart taosd +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql show databases +print rows: $rows +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +if $rows != 3 then + return -1 +endi +if $data00 != database then + return -1 +endi +if $data10 != DataBase then + return -1 +endi +if $data20 != information_schema then + return -1 +endi + +$dbCnt = 0 +while $dbCnt < 2 + if $dbCnt == 0 then + sql use `database` + else + sql use `DataBase` + endi + + $dbCnt = $dbCnt + 1 + + sql show stables + print rows: $rows + print $data00 $data01 + print $data10 $data11 + if $rows != 2 then + return -1 + endi + if $data00 != Stable then + if $data00 != stable then + return -1 + endi + endi + if $data10 != Stable then + if $data10 != stable then + return -1 + endi + endi + + sql show tables + print rows: $rows + print $data00 $data01 + print $data10 $data11 + print $data20 $data21 + print $data30 $data31 + if $rows != 4 then + return -1 + endi + + print =============== query data + sql select * from `table` + print rows: $rows + print $data00 $data01 $data02 $data03 + print $data10 $data11 $data12 $data13 + if $rows != 2 then + return -1 + endi + if $data01 != 10 then + return -1 + endi + if $data02 != table then + return -1 + endi + if $data03 != table then + return -1 + endi + + sql select * from `Table` + print rows: $rows + print $data00 $data01 $data02 $data03 + print $data10 $data11 $data12 $data13 + if $rows != 2 then + return -1 + endi + if $data01 != 20 then + return -1 + endi + if $data02 != Table then + return -1 + endi + if $data03 != Table then + return -1 + endi + + sql select * from `TAble` + print rows: $rows + print $data00 $data01 $data02 $data03 + print $data10 $data11 $data12 $data13 + if $rows != 2 then + return -1 + endi + if $data01 != 30 then + return -1 + endi + if $data02 != TAble then + return -1 + endi + if $data03 != TAble then + return -1 + endi + + sql select * from `TABle` + print rows: $rows + print $data00 $data01 $data02 $data03 + print $data10 $data11 $data12 $data13 + if $rows != 2 then + return -1 + endi + if $data01 != 40 then + return -1 + endi + if $data02 != TABle then + return -1 + endi + if $data03 != TABle then + return -1 + endi + + #print =============== query data from st, but not support select * from super table, waiting fix + #sql select count(*) from `stable` + #print rows: $rows + #print $data00 $data01 $data02 $data03 + #if $rows != 1 then + # return -1 + #endi + #if $data00 != 4 then + # return -1 + #endi + #sql select count(*) from `Stable` + #print rows: $rows + #print $data00 $data01 $data02 $data03 + #if $rows != 1 then + # return -1 + #endi + #if $data00 != 4 then + # return -1 + #endi + #sql select * from `stable` + #if $rows != 4 then + # return -1 + #endi + +endw + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/insert/basic0.sim b/tests/script/tsim/insert/basic0.sim index eb4780caac70747ff7745aa5601db9ffe8995854..da74eb95e81f95ea0dd7fcffab3706632a37ff88 100644 --- a/tests/script/tsim/insert/basic0.sim +++ b/tests/script/tsim/insert/basic0.sim @@ -7,7 +7,7 @@ sql connect print =============== create database sql create database d0 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -26,30 +26,37 @@ endi print =============== create child table sql create table ct1 using stb tags(1000) sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) sql show tables -if $rows != 2 then +if $rows != 3 then return -1 endi print =============== insert data, mode1: one row one table in sql print =============== insert data, mode1: mulit rows one table in sql -print =============== insert data, mode1: one rows mulit table in sql -print =============== insert data, mode1: mulit rows mulit table in sql +#print =============== insert data, mode1: one rows mulit table in sql +#print =============== insert data, mode1: mulit rows mulit table in sql sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) sql insert into ct2 values(now+0s, 10, 2.0, 3.0) -sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) -# after fix bug, modify sql_error to sql -sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) -sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) +sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +#sql insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) +#sql insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) + +sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) #=================================================================== #=================================================================== print =============== query data from child table sql select * from ct1 -if $rows != 4 then # after fix bug, modify 4 to 7 +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +if $rows != 4 then return -1 endi if $data01 != 10 then @@ -74,18 +81,19 @@ endi print =============== select count(*) from child table sql select count(*) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 if $data00 != 4 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 -print $data00 $data01 $data02 $data03 +print rows: $rows +print $data00 $data01 $data02 $data03 if $data00 != 4 then return -1 endi @@ -101,21 +109,41 @@ endi #print =============== select first(*)/first(column) from child table #sql select first(*) from ct1 -#sql select first(ts), first(c1), first(c2), first(c3) from ct1 +#print ====> select first(*) from ct1 +#print rows: $rows +#print $data00 $data01 $data02 $data03 + +sql select first(ts), first(c1), first(c2), first(c3) from ct1 +print ====> select first(ts), first(c1), first(c2), first(c3) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi print =============== select min(column) from child table sql select min(c1), min(c2), min(c3) from ct1 -print $data00 $data01 $data02 $data03 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != -13 then return -1 endi -if $data01 != 2.00000 then +if $data01 != -2.30000 then return -1 endi -if $data02 != 3.000000000 then +if $data02 != -3.300000000 then return -1 endi @@ -125,13 +153,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 11 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.10000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.100000000 then return -1 endi @@ -141,49 +169,53 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 46 then +if $data00 != -4 then return -1 endi -if $data01 != 8.599999905 then +if $data01 != -0.400000095 then return -1 endi -if $data02 != 12.600000000 then +if $data02 != -0.400000000 then return -1 endi -print =============== select column, from child table +print =============== select column without timestamp, from child table sql select c1, c2, c3 from ct1 +print rows: $rows print $data00 $data01 $data02 -#if $rows != 4 then -# return -1 -#endi -#if $data00 != 10 then -# return -1 -#endi -#if $data01 != 2.00000 then -# return -1 -#endi -#if $data02 != 3.000000000 then -# return -1 -#endi -#if $data10 != 11 then -# return -1 -#endi -#if $data11 != 2.10000 then -# return -1 -#endi -#if $data12 != 3.100000000 then -# return -1 -#endi -#if $data30 != 13 then -# return -1 -#endi -#if $data31 != 2.30000 then -# return -1 -#endi -#if $data32 != 3.300000000 then -# return -1 -#endi +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 +if $rows != 4 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != 11 then + return -1 +endi +if $data11 != 2.10000 then + return -1 +endi +if $data12 != 3.100000000 then + return -1 +endi +if $data30 != -13 then + return -1 +endi +if $data31 != -2.30000 then + return -1 +endi +if $data32 != -3.300000000 then + return -1 +endi #=================================================================== #=================================================================== @@ -192,21 +224,53 @@ print $data00 $data01 $data02 #if $rows != 4 then # return -1 #endi + #print =============== select count(*) from supter table #sql select count(*) from stb +#print $data00 $data01 $data02 #if $rows != 1 then # return -1 #endi -# -#print $data00 $data01 $data02 -#if $data00 != 8 then +#if $data00 != 9 then # return -1 #endi -# + +print =============== select count(column) from supter table +sql select ts, c1, c2, c3 from stb +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 +print $data50 $data51 $data52 $data53 +print $data60 $data61 $data62 $data63 +print $data70 $data71 $data72 $data73 +print $data80 $data81 $data82 $data83 +if $rows != 9 then + return -1 +endi + +# The order of data from different sub tables in the super table is random, +# so this detection may fail randomly +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi + #print =============== select count(column) from supter table #sql select count(ts), count(c1), count(c2), count(c3) from stb +#print rows: $rows #print $data00 $data01 $data02 $data03 -#if $data00 != 8 then +#print $data10 $data11 $data12 $data13 +#print $data20 $data21 $data22 $data23 +#print $data30 $data31 $data32 $data33 +#if $data00 != 9 then # return -1 #endi #if $data01 != 8 then @@ -219,7 +283,6 @@ print $data00 $data01 $data02 # return -1 #endi - #=================================================================== #=================================================================== @@ -227,9 +290,31 @@ print =============== stop and restart taosd, then again do query above system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -sleep 2000 +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +print =============== query data from child table sql select * from ct1 -if $rows != 4 then # after fix bug, modify 4 to 7 +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +if $rows != 4 then return -1 endi if $data01 != 10 then @@ -254,18 +339,19 @@ endi print =============== select count(*) from child table sql select count(*) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 if $data00 != 4 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 -print $data00 $data01 $data02 $data03 +print rows: $rows +print $data00 $data01 $data02 $data03 if $data00 != 4 then return -1 endi @@ -281,21 +367,41 @@ endi #print =============== select first(*)/first(column) from child table #sql select first(*) from ct1 -#sql select first(ts), first(c1), first(c2), first(c3) from ct1 +#print ====> select first(*) from ct1 +#print rows: $rows +#print $data00 $data01 $data02 $data03 + +sql select first(ts), first(c1), first(c2), first(c3) from ct1 +print ====> select first(ts), first(c1), first(c2), first(c3) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi print =============== select min(column) from child table sql select min(c1), min(c2), min(c3) from ct1 -print $data00 $data01 $data02 $data03 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != -13 then return -1 endi -if $data01 != 2.00000 then +if $data01 != -2.30000 then return -1 endi -if $data02 != 3.000000000 then +if $data02 != -3.300000000 then return -1 endi @@ -305,13 +411,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 11 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.10000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.100000000 then return -1 endi @@ -321,14 +427,114 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 46 then +if $data00 != -4 then + return -1 +endi +if $data01 != -0.400000095 then return -1 endi -if $data01 != 8.599999905 then +if $data02 != -0.400000000 then return -1 endi -if $data02 != 12.600000000 then + +print =============== select column without timestamp, from child table +sql select c1, c2, c3 from ct1 +print rows: $rows +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 +if $rows != 4 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then return -1 endi +if $data10 != 11 then + return -1 +endi +if $data11 != 2.10000 then + return -1 +endi +if $data12 != 3.100000000 then + return -1 +endi +if $data30 != -13 then + return -1 +endi +if $data31 != -2.30000 then + return -1 +endi +if $data32 != -3.300000000 then + return -1 +endi +#=================================================================== +#=================================================================== + +#print =============== query data from stb +#sql select * from stb +#if $rows != 4 then +# return -1 +#endi + +#print =============== select count(*) from supter table +#sql select count(*) from stb +#print $data00 $data01 $data02 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 9 then +# return -1 +#endi + +print =============== select count(column) from supter table +sql select ts, c1, c2, c3 from stb +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 +print $data50 $data51 $data52 $data53 +print $data60 $data61 $data62 $data63 +print $data70 $data71 $data72 $data73 +print $data80 $data81 $data82 $data83 +if $rows != 9 then + return -1 +endi +# The order of data from different sub tables in the super table is random, +# so this detection may fail randomly +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi + +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 8 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/insert/basic1.sim b/tests/script/tsim/insert/basic1.sim index 3fc635532ac409000988808249d0310b7096c5e7..653a44a18a862be0924ffafeff86ec537a75604d 100644 --- a/tests/script/tsim/insert/basic1.sim +++ b/tests/script/tsim/insert/basic1.sim @@ -7,7 +7,7 @@ sql connect print =============== create database sql create database d1 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -46,11 +46,17 @@ sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1 print =============== query data sql select * from c1 +print rows: $rows +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 if $rows != 4 then return -1 endi -if $data01 != true then +if $data01 != 1 then + print expect 1, actual: $data01 return -1 endi @@ -62,24 +68,45 @@ if $data03 != -2 then return -1 endi -print =============== query data from st -sql select * from st -if $rows != 4 then - return -1 -endi +print =============== query data from st, but not support select * from super table, waiting fix +#sql select * from st +#if $rows != 4 then +# return -1 +#endi print =============== stop and restart taosd system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -sleep 2000 +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + print =============== query data sql select * from c1 +print rows: $rows +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 if $rows != 4 then return -1 endi -if $data01 != true then +if $data01 != 1 then return -1 endi @@ -91,4 +118,10 @@ if $data03 != -2 then return -1 endi +print =============== query data from st, but not support select * from super table, waiting fix +#sql select * from st +#if $rows != 4 then +# return -1 +#endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/insert/null.sim b/tests/script/tsim/insert/null.sim new file mode 100644 index 0000000000000000000000000000000000000000..fbaef8cc94105b06c0b44b67ea23e57a2346c9cd --- /dev/null +++ b/tests/script/tsim/insert/null.sim @@ -0,0 +1,474 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d0 +sql show databases +if $rows != 2 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use d0 + +print =============== create super table, include column type for count/sum/min/max/first +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double, c4 bigint) tags (t1 int unsigned) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) + +sql show tables +if $rows != 3 then + return -1 +endi + +print =============== insert data, include NULL +sql insert into ct1 values (now+0s, 10, 2.0, 3.0, 90)(now+1s, NULL, NULL, NULL, NULL)(now+2s, NULL, 2.1, 3.1, 91)(now+3s, 11, NULL, 3.2, 92)(now+4s, 12, 2.2, NULL, 93)(now+5s, 13, 2.3, 3.3, NULL) +sql insert into ct1 values (now+6s, NULL, 2.4, 3.4, 94) +sql insert into ct1 values (now+7s, 14, NULL, 3.5, 95) +sql insert into ct1 values (now+8s, 15, 2.5, NULL, 96) +sql insert into ct1 values (now+9s, 16, 2.6, 3.6, NULL) +sql insert into ct1 values (now+10s, NULL, NULL, NULL, NULL) +sql insert into ct1 values (now+11s, -2147483647, 2.7, 3.7, 97) + +#=================================================================== +#=================================================================== +print =============== query data from child table +sql select * from ct1 +print ===> select * from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 12 then + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi +#if $data41 != -14 then +# return -1 +#endi +#if $data42 != -2.40000 then +# return -1 +#endi +#if $data43 != -3.400000000 then +# return -1 +#endi + +print =============== select count(*) from child table +sql select count(*) from ct1 +print ===> select count(*) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != 12 then + return -1 +endi + +print =============== select count(column) from child table +sql select count(ts), count(c1), count(c2), count(c3) from ct1 +print ===> select count(ts), count(c1), count(c2), count(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $data00 != 12 then + return -1 +endi +if $data01 != 8 then + return -1 +endi +if $data02 != 8 then + return -1 +endi +if $data03 != 8 then + return -1 +endi + +#print =============== select first(*)/first(column) from child table +#sql select first(*) from ct1 +#sql select first(ts), first(c1), first(c2), first(c3) from ct1 + +print =============== select min(column) from child table +sql select min(c1), min(c2), min(c3) from ct1 +print ===> select min(c1), min(c2), min(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != -2147483647 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi + +print =============== select max(column) from child table +sql select max(c1), max(c2), max(c3) from ct1 +print ===> select max(c1), max(c2), max(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != 16 then + return -1 +endi +if $data01 != 2.70000 then + return -1 +endi +if $data02 != 3.700000000 then + return -1 +endi + +print =============== select sum(column) from child table +sql select sum(c1), sum(c2), sum(c3) from ct1 +print ===> select sum(c1), sum(c2), sum(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != -2147483556 then + return -1 +endi +if $data01 != 18.799999952 then + return -1 +endi +if $data02 != 26.800000000 then + return -1 +endi + +print =============== select column, from child table +sql select c1, c2, c3 from ct1 +print ===> select c1, c2, c3 from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 12 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data11 != NULL then + return -1 +endi +if $data12 != NULL then + return -1 +endi +if $data30 != 11 then + return -1 +endi +if $data31 != NULL then + return -1 +endi +if $data32 != 3.200000000 then + return -1 +endi +if $data90 != 16 then + return -1 +endi +if $data91 != 2.60000 then + return -1 +endi +if $data92 != 3.600000000 then + return -1 +endi +#=================================================================== +#=================================================================== + +#print =============== query data from stb +#sql select * from stb +#print ===> +#print ===> rows: $rows +#print ===> rows0: $data00 $data01 $data02 $data03 $data04 +#if $rows != 4 then +# return -1 +#endi +#print =============== select count(*) from supter table +#sql select count(*) from stb +#print $data00 $data01 $data02 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 12 then +# return -1 +#endi + +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 12 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi + +#=================================================================== +#=================================================================== + +print =============== stop and restart taosd, then again do query above +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +print ===> waiting dnode ready +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +#=================================================================== +#=================================================================== +print =============== query data from child table +sql select * from ct1 +print ===> select * from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 12 then + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi +#if $data41 != -14 then +# return -1 +#endi +#if $data42 != -2.40000 then +# return -1 +#endi +#if $data43 != -3.400000000 then +# return -1 +#endi + +print =============== select count(*) from child table +sql select count(*) from ct1 +print ===> select count(*) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != 12 then + return -1 +endi + +print =============== select count(column) from child table +sql select count(ts), count(c1), count(c2), count(c3) from ct1 +print ===> select count(ts), count(c1), count(c2), count(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $data00 != 12 then + return -1 +endi +if $data01 != 8 then + return -1 +endi +if $data02 != 8 then + return -1 +endi +if $data03 != 8 then + return -1 +endi + +#print =============== select first(*)/first(column) from child table +#sql select first(*) from ct1 +#sql select first(ts), first(c1), first(c2), first(c3) from ct1 + +print =============== select min(column) from child table +sql select min(c1), min(c2), min(c3) from ct1 +print ===> select min(c1), min(c2), min(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != -2147483647 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi + +print =============== select max(column) from child table +sql select max(c1), max(c2), max(c3) from ct1 +print ===> select max(c1), max(c2), max(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != 16 then + return -1 +endi +if $data01 != 2.70000 then + return -1 +endi +if $data02 != 3.700000000 then + return -1 +endi + +print =============== select sum(column) from child table +sql select sum(c1), sum(c2), sum(c3) from ct1 +print ===> select sum(c1), sum(c2), sum(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + return -1 +endi +if $data00 != -2147483556 then + return -1 +endi +if $data01 != 18.799999952 then + return -1 +endi +if $data02 != 26.800000000 then + return -1 +endi + +print =============== select column, from child table +sql select c1, c2, c3 from ct1 +print ===> select c1, c2, c3 from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 12 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data11 != NULL then + return -1 +endi +if $data12 != NULL then + return -1 +endi +if $data30 != 11 then + return -1 +endi +if $data31 != NULL then + return -1 +endi +if $data32 != 3.200000000 then + return -1 +endi +if $data90 != 16 then + return -1 +endi +if $data91 != 2.60000 then + return -1 +endi +if $data92 != 3.600000000 then + return -1 +endi +#=================================================================== +#=================================================================== + +#print =============== query data from stb +#sql select * from stb +#print ===> +#print ===> rows: $rows +#print ===> rows0: $data00 $data01 $data02 $data03 $data04 +#if $rows != 4 then +# return -1 +#endi +#print =============== select count(*) from supter table +#sql select count(*) from stb +#print $data00 $data01 $data02 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 12 then +# return -1 +#endi + +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 12 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 8034a27b9f074dfa5389ebe80c51da0a480a8b15..f188dff1ba91cb0311a58f992d00f3a4ef4430ae 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -1,292 +1,324 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 50 +sleep 500 sql connect print =============== create database sql create database d0 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi -print $data00 $data01 $data02 - sql use d0 - -sql create table vehicle (ts timestamp, s int) tags (t1 int) +print =============== create super table and child table +sql create table stb (ts timestamp, tbcol int) tags (t1 int) sql show stables +print $rows $data00 $data01 $data02 if $rows != 1 then return -1 endi -sql create table car using vehicle tags(1000) -#sql create table ct1 using vehicle tags (1000) -sql show tables; -if $rows != 1 then - print rows: $rows +sql create table ct1 using stb tags ( 1 ) +sql create table ct2 using stb tags ( 2 ) +sql create table ct3 using stb tags ( 3 ) +sql create table ct4 using stb tags ( 4 ) +sql show tables +print $rows $data00 $data10 $data20 +if $rows != 4 then return -1 endi -sql insert into ct1 values('2019-01-01 00:00:00', 1) -sql insert into car values('2019-05-13 12:00:00', 1) -sql insert into car values('2019-12-31 23:59:59', 1) -sql insert into car values('2020-01-01 12:00:00', 1) -sql insert into car values('2020-01-02 12:00:00', 1) -sql insert into car values('2020-01-03 12:00:00', 1) -sql insert into car values('2020-01-04 12:00:00', 1) -sql insert into car values('2020-01-05 12:00:00', 1) -sql insert into car values('2020-01-31 12:00:00', 1) -sql insert into car values('2020-02-01 12:00:00', 1) -sql insert into car values('2020-02-02 12:00:00', 1) -sql insert into car values('2020-02-29 12:00:00', 1) -sql insert into car values('2020-03-01 12:00:00', 1) -sql insert into car values('2020-03-02 12:00:00', 1) -sql insert into car values('2020-03-15 12:00:00', 1) -sql insert into car values('2020-03-31 12:00:00', 1) -sql insert into car values('2020-05-01 12:00:00', 1) - -sql select count(*) from car interval(1n, 10d) -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 1) -# tdSql.checkData(2, 1, 6) -# tdSql.checkData(3, 1, 3) -# tdSql.checkData(4, 1, 3) -# tdSql.checkData(5, 1, 2) -# tdSql.checkData(6, 1, 1) -if $rows != 17 then +print =============== insert data into child table ct1 (s) +sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:06.000', 2 ) +sql insert into ct1 values ( '2022-01-01 01:01:10.000', 3 ) +sql insert into ct1 values ( '2022-01-01 01:01:16.000', 4 ) +sql insert into ct1 values ( '2022-01-01 01:01:20.000', 5 ) +sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6 ) +sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7 ) +sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8 ) + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 5 then return -1 -endi -if $data00 != 1 then +endi +if $data00 != 1 then return -1 -endi -if $data10 != 1 then +endi +if $data40 != 1 then return -1 -endi -if $data20 != 6 then +endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 5 then return -1 -endi -if $data30 != 3 then +endi +if $data00 != 1 then return -1 -endi -if $data40 != 3 then +endi +if $data40 != 1 then return -1 -endi -if $data50 != 2 then +endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(5s) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(5s) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 +print ===> rows8: $data80 $data81 $data82 $data83 $data84 +if $rows != 9 then return -1 -endi -if $data60 != 1 then +endi +if $data00 != 1 then return -1 -endi - -return - -sql select count(*) from car interval(1n, 10d) order by ts desc -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 2) -# tdSql.checkData(2, 1, 3) -# tdSql.checkData(3, 1, 3) -# tdSql.checkData(4, 1, 6) -# tdSql.checkData(5, 1, 1) -# tdSql.checkData(6, 1, 1) -# -sql select count(*) from car interval(2n, 5d) -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 1) -# tdSql.checkData(2, 1, 6) -# tdSql.checkData(3, 1, 6) -# tdSql.checkData(4, 1, 3) - -sql select count(*) from car interval(2n) order by ts desc -# tdSql.checkData(0, 1, 3) -# tdSql.checkData(1, 1, 6) -# tdSql.checkData(2, 1, 6) -# tdSql.checkData(3, 1, 1) -# tdSql.checkData(4, 1, 1) -# -sql select count(*) from car interval(1y, 1n) -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 8) -# tdSql.checkData(2, 1, 8) -# -sql select count(*) from car interval(1y, 2n) -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 11) -# tdSql.checkData(2, 1, 5) - -sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) -# tdSql.checkData(0, 1, 6) -# tdSql.checkData(1, 1, 9) - - - - - - - - - - - - -sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) - -print ====== start create child tables and insert data -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = 0 - while $x < $rowNum - $cc = $x * 60000 - $ms = 1601481600000 + $cc - - sql insert into $tb values ($ms , $x ) - $x = $x + 1 - endw - - $i = $i + 1 -endw - -print =============== step2 -$i = 1 -$tb = $tbPrefix . $i +endi +if $data70 != 2 then + return -1 +endi +if $data80 != 1 then + return -1 +endi -sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) -print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) -print ===> $rows $data01 $data05 -if $rows != $rowNum then +print =============== insert data into child table ct2 (d) +sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2 ) +sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3 ) +sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4 ) +sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5 ) +sql insert into ct2 values ( '2022-01-03 10:00:01.000', 6 ) +sql insert into ct2 values ( '2022-01-03 20:00:01.000', 7 ) + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 4 then return -1 -endi -if $data00 != 1 then +endi +if $data00 != 1 then return -1 -endi -if $data04 != 1 then +endi +if $data10 != 2 then return -1 endi -#print =============== step3 -#$cc = 4 * 60000 -#$ms = 1601481600000 + $cc -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) -#print ===> $rows $data01 $data05 -#if $rows != 5 then +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 +#if $rows != 8 then # return -1 -#endi -#if $data00 != 1 then +#endi +#if $data00 != 1 then # return -1 -#endi -#if $data04 != 1 then +#endi +#if $data10 != 2 then # return -1 -#endi - -#print =============== step4 -#$cc = 40 * 60000 -#$ms = 1601481600000 + $cc - -#$cc = 1 * 60000 -#$ms2 = 1601481600000 - $cc - -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) -#print ===> $rows $data01 $data05 -#if $rows != 20 then +#endi +#if $data20 != 2 then # return -1 -#endi -#if $data00 != 1 then +#endi +#if $data30 != 2 then # return -1 -#endi -#if $data04 != 1 then +#endi +#if $data40 != 2 then # return -1 -#endi - -#print =============== step5 -#$cc = 40 * 60000 -#$ms = 1601481600000 + $cc - -#$cc = 1 * 60000 -#$ms2 = 1601481600000 - $cc - -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) -#print ===> $rows $data21 $data25 -#if $rows != 42 then +#endi +#if $data50 != 2 then # return -1 -#endi -#if $data20 != 1 then +#endi +#if $data60 != 2 then # return -1 -#endi -#if $data24 != 1 then +#endi +#if $data70 != 1 then # return -1 #endi -#print =============== step6 -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) -#print ===> $rows $data11 -#if $rows != 20 then +print =============== insert data into child table ct3 (n) +sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL ); +sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1 ); +sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2 ); +sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3 ); +sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4 ); +sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5 ); +sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6 ); +sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7 ); +sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8 ); + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +#if $rows != 5 then # return -1 -#endi -#if $data11 != 10 then +#endi +#if $data00 != 1 then # return -1 -#endi - -#print =============== step7 -#$cc = 4 * 60000 -#$ms = 1601481600000 + $cc -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) -#print ===> $rows $data11 -#if $rows != 5 then +#endi +#if $data40 != 1 then # return -1 -#endi -#if $data11 != 10 then +#endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +#if $rows != 5 then # return -1 -#endi - -#print =============== step8 -#$cc = 40 * 60000 -#$ms1 = 1601481600000 + $cc -# -#$cc = 1 * 60000 -#$ms2 = 1601481600000 - $cc -# -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) -#print ===> $rows $data11 -#if $rows != 20 then +#endi +#if $data00 != 1 then # return -1 -#endi -#if $data11 != 10 then +#endi +#if $data40 != 1 then # return -1 -#endi -# -#print =============== step9 -#$cc = 40 * 60000 -#$ms1 = 1601481600000 + $cc -# -#$cc = 1 * 60000 -#$ms2 = 1601481600000 - $cc -# -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) -#print ===> $rows $data11 -#if $rows != 42 then +#endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 +#if $rows != 8 then # return -1 -#endi -#if $data11 != 10 then +#endi +#if $data00 != 2 then # return -1 -#endi - -print =============== clear -#sql drop database $db -#sql show databases -#if $rows != 0 then +#endi +#if $data70 != 1 then # return -1 -#endi +#endi + + +print =============== insert data into child table ct4 (y) +sql insert into ct4 values ( '2020-10-21 01:01:01.000', 1 ) +sql insert into ct4 values ( '2020-12-31 01:01:01.000', 2 ) +sql insert into ct4 values ( '2021-01-01 01:01:06.000', 3 ) +sql insert into ct4 values ( '2021-05-07 01:01:10.000', 4 ) +sql insert into ct4 values ( '2021-09-30 01:01:16.000', 5 ) +sql insert into ct4 values ( '2022-02-01 01:01:20.000', 6 ) +sql insert into ct4 values ( '2022-10-28 01:01:26.000', 7 ) +sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 ) +sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 ) + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +#if $rows != 5 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data40 != 1 then +# return -1 +#endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +#if $rows != 5 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data40 != 1 then +# return -1 +#endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 +#if $rows != 8 then +# return -1 +#endi +#if $data00 != 2 then +# return -1 +#endi +#if $data70 != 1 then +# return -1 +#endi + + + + +#sql select count(*) from car interval(1n, 10d) order by ts desc +#sql select count(*) from car interval(2n, 5d) +#sql select count(*) from car interval(2n) order by ts desc +#sql select count(*) from car interval(1y, 1n) +#sql select count(*) from car interval(1y, 2n) +#sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) + + + + + + #system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 47be71bdc6f54a81c94066c1cd62eb393f030b5c..6dd0a9537e0de9452f5d13659eb4372faf871e40 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wal -v 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 +sleep 500 sql connect $dbPrefix = m_in_db diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim new file mode 100644 index 0000000000000000000000000000000000000000..cc926861422518d4cfb5fffcd5f2f9d273950f22 --- /dev/null +++ b/tests/script/tsim/show/basic.sim @@ -0,0 +1,215 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start + +# after mnode support, del sleep 2000, and check dnode status +sleep 2000 +sql connect + +#$loop_cnt = 0 +#check_dnode_ready: +# $loop_cnt = $loop_cnt + 1 +# sleep 200 +# if $loop_cnt == 10 then +# print ====> dnode not ready! +# return -1 +# endi +#sql show dnodes +#print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +#if $data00 != 1 then +# return -1 +#endi +#if $data04 != ready then +# goto check_dnode_ready +#endi +#sql connect + +# select */column from information_schema.xxxx; xxxx include: +# dnodes, mnodes, modules, qnodes, +# user_databases, user_functions, user_indexes, user_stables, user_streams, +# user_tables, user_table_distributed, user_users, vgroups, + +print =============== add dnode2 into cluster +sql create dnode $hostname port 7200 + +print =============== create database, stable, table +sql create database db vgroups 3 +sql use db +sql create table stb (ts timestamp, c int) tags (t int) +sql create table t0 using stb tags (0) + +print =============== run show xxxx +sql show dnodes +if $rows != 2 then + return -1 +endi + +sql show mnodes +if $rows != 1 then + return -1 +endi +#sql show modules +#sql show qnodes +sql show databases +if $rows != 2 then + return -1 +endi +#sql show functions + +#sql show indexes +sql show stables +if $rows != 1 then + return -1 +endi +#sql show streams, +sql show tables +if $rows != 1 then + return -1 +endi +#sql show user_table_distributed +sql show users +if $rows != 1 then + return -1 +endi +sql show vgroups +if $rows != 3 then + return -1 +endi + +print =============== run select * from information_schema.xxxx +sql select * from information_schema.`dnodes` +if $rows != 2 then + return -1 +endi +sql select * from information_schema.`mnodes` +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.`modules` +#sql select * from information_schema.`qnodes` +sql select * from information_schema.user_databases +if $rows != 2 then + return -1 +endi +#sql select * from information_schema.user_functions +#sql select * from information_schema.user_indexes +sql select * from information_schema.user_stables +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.user_streams +sql select * from information_schema.user_tables +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.user_table_distributed +sql select * from information_schema.user_users +if $rows != 1 then + return -1 +endi +sql select * from information_schema.`vgroups` +if $rows != 3 then + return -1 +endi + +print ==== stop dnode1 and dnode2, and restart dnodes +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +print ==== again run show / select of above +print =============== run show xxxx +sql show dnodes +if $rows != 2 then + return -1 +endi + +sql show mnodes +if $rows != 1 then + return -1 +endi +#sql show modules +#sql show qnodes +sql show databases +if $rows != 2 then + return -1 +endi +#sql show functions + +#sql show indexes +sql show stables +if $rows != 1 then + return -1 +endi +#sql show streams, +sql show tables +if $rows != 1 then + return -1 +endi +#sql show user_table_distributed +sql show users +if $rows != 1 then + return -1 +endi +sql show vgroups +if $rows != 3 then + return -1 +endi + +print =============== run select * from information_schema.xxxx +sql select * from information_schema.`dnodes` +if $rows != 2 then + return -1 +endi +sql select * from information_schema.`mnodes` +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.`modules` +#sql select * from information_schema.`qnodes` +sql select * from information_schema.user_databases +if $rows != 2 then + return -1 +endi +#sql select * from information_schema.user_functions +#sql select * from information_schema.user_indexes +sql select * from information_schema.user_stables +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.user_streams +sql select * from information_schema.user_tables +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.user_table_distributed +sql select * from information_schema.user_users +if $rows != 1 then + return -1 +endi +sql select * from information_schema.`vgroups` +if $rows != 3 then + return -1 +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index 257c26415994fa245fa42e285c667598d6615d1c..bdb49cc9cdcd79739df3e547f6e98d8b052c6f0f 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -3,10 +3,33 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect +#=========== TD-14042 start +sql create database db1; +sql create database db2; +sql create database db3; + +sql use db1; +sql create table st1 (ts timestamp, i int) tags (j int); +sql create table tb1 using st1 tags(1); + +sql use db2; +sql create table st2 (ts timestamp, i int) tags (j int); +sql create table tb2 using st2 tags(1); + +sql use db3; +sql create table st3 (ts timestamp, i int) tags (j int); +sql create table tb3 using st3 tags(1); + +sql show tables +if $rows != 1 then + return -1 +endi +#=========== TD-14042 end + print =============== create database sql create database d1 sql show databases -if $rows != 1 then +if $rows != 5 then return -1 endi @@ -187,12 +210,31 @@ if $rows != 21 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT - system sh/exec.sh -n dnode1 -s start -sleep 2000 +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + print =============== query data sql select * from c1 +print rows: $rows +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 if $rows != 3 then print $rows return -1 diff --git a/tests/script/tsim/testCaseSuite.sim b/tests/script/tsim/testCaseSuite.sim new file mode 100644 index 0000000000000000000000000000000000000000..4b700d2ca4f6f169457f72e9d819dab05c83337b --- /dev/null +++ b/tests/script/tsim/testCaseSuite.sim @@ -0,0 +1,24 @@ + +run tsim/user/basic1.sim + +run tsim/db/basic1.sim +run tsim/db/basic6.sim +run tsim/db/basic7.sim +run tsim/db/error1.sim + +run tsim/dnode/basic1.sim + +run tsim/insert/basic0.sim +run tsim/insert/basic1.sim +run tsim/insert/backquote.sim +run tsim/insert/null.sim + +run tsim/query/interval.sim +#run tsim/query/interval-offset.sim # TD-14266 + +run tsim/show/basic.sim + +run tsim/table/basic1.sim + +run tsim/tmq/basic.sim + diff --git a/tests/script/tsim/tmq/basic.sim b/tests/script/tsim/tmq/basic.sim new file mode 100644 index 0000000000000000000000000000000000000000..876cf7e266d29cb06f4914cafde6c1c38bd5d8e1 --- /dev/null +++ b/tests/script/tsim/tmq/basic.sim @@ -0,0 +1,79 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wal -v 1 +system sh/exec.sh -n dnode1 -s start +sleep 500 +sql connect + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 100 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +#root@trd02 /data2/dnode $ tmq_demo --help +#Used to tmq_demo +# -c Configuration directory, default is +# -d The name of the database to be created, default is tmqdb +# -s The name of the super table to be created, default is stb +# -f The file of result, default is ./tmqResult.txt +# -w The path of vnode of wal, default is /data2/dnode/data/vnodes/vnode2/wal +# -t numOfThreads, default is 1 +# -n numOfTables, default is 1 +# -v numOfVgroups, default is 1 +# -a runMode, default is 0 +# -l numOfColumn, default is 1 +# -q ratio, default is 1.000000 +# -b batchNumOfRow, default is 1 +# -r totalRowsOfPerTbl, default is 10000 +# -m startTimestamp, default is 1640966400000 [2022-01-01 00:00:00] +# -g showMsgFlag, default is 0 +# +print cmd===> system_content ../../debug/tests/test/c/tmq_demo -sim 1 -b 100 -c ../../sim/tsim/cfg -w ../../sim/dnode1/data/vnode/vnode4/wal +system_content ../../debug/tests/test/c/tmq_demo -sim 1 -b 100 -c ../../sim/tsim/cfg -w ../../sim/dnode1/data/vnode/vnode4/wal +print cmd result----> $system_content +if $system_content != @{consume success: 100}@ then + print not match in pos000 +endi + +sql show databases +print ===> $rows $data00 $data01 $data02 $data03 +if $rows != 2 then + return -1 +endi +if $data00 != tmqdb then + return -1 +endi + +sql use tmqdb +sql show tables +print ===> $rows $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != stb0 then + return -1 +endi + +sql select count(*) from stb0 +print ===> $rows $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 10000 then + return -1 +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/user/basic1.sim b/tests/script/tsim/user/basic1.sim index 33c4dc96d41c76e9ebe2661d806cb1d0ac6b304a..7af5ba8d001bd9795378f53b91663d671afc351f 100644 --- a/tests/script/tsim/user/basic1.sim +++ b/tests/script/tsim/user/basic1.sim @@ -71,4 +71,4 @@ print $data10 $data11 $data22 print $data20 $data11 $data22 print $data30 $data31 $data32 -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 57fbd4c9c1a55707f66f813fc17272e9526ffab0..4fca7d6245b6e5dcf18f4fa2103e6204ec4e6188 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -49,7 +49,7 @@ typedef struct { int64_t startMs; int64_t maxDelay; int64_t minDelay; - pthread_t thread; + TdThread thread; } SThreadInfo; // void parseArgument(int32_t argc, char *argv[]); @@ -173,7 +173,7 @@ void showTables() { void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; - char *qstr = malloc(batchNumOfTbl * batchNumOfRow * 128); + char *qstr = taosMemoryMalloc(batchNumOfTbl * batchNumOfRow * 128); int32_t code = 0; TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); @@ -284,7 +284,7 @@ void *threadFunc(void *param) { } taos_close(con); - free(qstr); + taosMemoryFree(qstr); return 0; } @@ -400,10 +400,10 @@ int32_t main(int32_t argc, char *argv[]) { pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); // int64_t numOfTablesPerThread = numOfTables / numOfThreads; // numOfTables = numOfTablesPerThread * numOfThreads; @@ -430,12 +430,12 @@ int32_t main(int32_t argc, char *argv[]) { pInfo[i].minDelay = INT64_MAX; strcpy(pInfo[i].dbName, dbName); strcpy(pInfo[i].stbName, stbName); - pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); } taosMsleep(300); for (int32_t i = 0; i < numOfThreads; i++) { - pthread_join(pInfo[i].thread, NULL); + taosThreadJoin(pInfo[i].thread, NULL); } int64_t maxDelay = 0; @@ -465,6 +465,6 @@ int32_t main(int32_t argc, char *argv[]) { numOfThreads, NC); } - pthread_attr_destroy(&thattr); - free(pInfo); + taosThreadAttrDestroy(&thattr); + taosMemoryFree(pInfo); } diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 609f8d6b694d69218b8a54eddd4cbf435628d37d..205bc0a639859fb9789c415d304fb1ee24832aea 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -58,6 +58,7 @@ typedef struct { int32_t totalRowsOfPerTbl; int64_t startTimestamp; int32_t showMsgFlag; + int32_t simCase; int32_t totalRowsOfT2; } SConfInfo; @@ -65,11 +66,11 @@ typedef struct { static SConfInfo g_stConfInfo = { "tmqdb", "stb", - "./tmqResult.txt", // output_file - "/data2/dnode/data/vnode/vnode2/wal", + "./tmqResult.txt", // output_file + "/data2/dnode/data/vnode/vnode2/wal", 1, // threads 1, // tables - 1, // vgroups + 1, // vgroups 0, // run mode 1, // columns 1, // ratio @@ -77,6 +78,7 @@ static SConfInfo g_stConfInfo = { 10000, // total rows for per table 0, // 2020-01-01 00:00:00.000 0, // show consume msg switch + 0, // if run in sim case 10000, }; @@ -117,6 +119,8 @@ static void printHelp() { printf("%s%s%s%" PRId64 "\n", indent, indent, "startTimestamp, default is ", g_stConfInfo.startTimestamp); printf("%s%s\n", indent, "-g"); printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-sim"); + printf("%s%s%s%d\n", indent, indent, "simCase, default is ", g_stConfInfo.simCase); exit(EXIT_SUCCESS); } @@ -160,14 +164,17 @@ void parseArgument(int32_t argc, char *argv[]) { g_stConfInfo.startTimestamp = atol(argv[++i]); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-sim") == 0) { + g_stConfInfo.simCase = atol(argv[++i]); } else { - pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); + printf("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); } } g_stConfInfo.totalRowsOfT2 = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.ratio; +#if 0 pPrint("%s configDir:%s %s", GREEN, configDir, NC); pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); pPrint("%s stbName:%s %s", GREEN, g_stConfInfo.stbName, NC); @@ -184,6 +191,7 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s totalRowsOfT2:%d %s", GREEN, g_stConfInfo.totalRowsOfT2, NC); pPrint("%s startTimestamp:%" PRId64" %s", GREEN, g_stConfInfo.startTimestamp, NC); pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); +#endif } static int running = 1; @@ -267,7 +275,7 @@ int32_t init_env() { taos_free_result(pRes); // create row value - g_pRowValue = (char*)calloc(1, g_stConfInfo.numOfColumn * 16 + 128); + g_pRowValue = (char*)taosMemoryCalloc(1, g_stConfInfo.numOfColumn * 16 + 128); if (NULL == g_pRowValue) { return -1; } @@ -306,8 +314,9 @@ int32_t init_env() { } //const char* sql = "select * from tu1"; - sprintf(sqlStr, "select * from %s%d", g_stConfInfo.stbName, 0); - pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr)); + sprintf(sqlStr, "create topic test_stb_topic_1 as select * from %s0", g_stConfInfo.stbName); + /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/ + pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -428,15 +437,21 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog double consumeTime = (double)(endTime - startTime) / 1000000; if (batchCnt != totalMsgs) { - pPrint("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); + printf("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); + exit(-1); } - pPrint("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + if (0 == g_stConfInfo.simCase) { + printf("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + } else { + printf("{consume success: %d}", totalMsgs); + } taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.2f| %10.2f |\n", batchCnt, consumeTime, (double)batchCnt / consumeTime, (double)walLogSize / (1024 * 1024.0) / consumeTime, (double)walLogSize / 1024.0 / batchCnt); err = tmq_consumer_close(tmq); if (err) { fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + exit(-1); } } @@ -457,7 +472,7 @@ int32_t syncWriteData() { taos_free_result(pRes); char* buffer = NULL; - buffer = (char*)malloc(MAX_SQL_STR_LEN); + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); if (NULL == buffer) { return -1; } @@ -490,7 +505,7 @@ int32_t syncWriteData() { int code = queryDB(pConn, buffer); if (0 != code){ fprintf(stderr, "insert data error!\n"); - tfree(buffer); + taosMemoryFreeClear(buffer); return -1; } @@ -502,7 +517,7 @@ int32_t syncWriteData() { } } } - tfree(buffer); + taosMemoryFreeClear(buffer); return totalMsgs; } @@ -524,7 +539,7 @@ int32_t syncWriteDataByRatio() { taos_free_result(pRes); char* buffer = NULL; - buffer = (char*)malloc(MAX_SQL_STR_LEN); + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); if (NULL == buffer) { return -1; } @@ -582,7 +597,7 @@ int32_t syncWriteDataByRatio() { int code = queryDB(pConn, buffer); if (0 != code){ fprintf(stderr, "insert data error!\n"); - tfree(buffer); + taosMemoryFreeClear(buffer); return -1; } @@ -596,7 +611,7 @@ int32_t syncWriteDataByRatio() { } } pPrint("expect insert rows: T1[%d] T2[%d], actual insert rows: T1[%d] T2[%d]\n", g_stConfInfo.totalRowsOfPerTbl, g_stConfInfo.totalRowsOfT2, insertedOfT1, insertedOfT2); - tfree(buffer); + taosMemoryFreeClear(buffer); return totalMsgs; } @@ -678,12 +693,17 @@ int main(int32_t argc, char *argv[]) { walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); if (walLogSize <= 0) { - pError("vnode2/wal size incorrect!"); + printf("vnode2/wal size incorrect!"); + exit(-1); } else { - pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); + if (0 == g_stConfInfo.simCase) { + pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); + } } - - pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, totalMsgs, seconds, rowsSpeed, msgsSpeed); + + if (0 == g_stConfInfo.simCase) { + pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, totalMsgs, seconds, rowsSpeed, msgsSpeed); + } taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.3f ", totalMsgs, seconds, msgsSpeed, (double)walLogSize/(1024 * 1024.0)); } @@ -699,7 +719,7 @@ int main(int32_t argc, char *argv[]) { perf_loop(tmq, topic_list, totalMsgs, walLogSize); - tfree(g_pRowValue); + taosMemoryFreeClear(g_pRowValue); taosFprintfFile(g_fp, "\n"); taosCloseFile(&g_fp); return 0; diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h index 24619a4ad59fdafc8c3bf7e13d7b01ca6ba5e02a..1e2190e3086af93e0e1dccd4bc1fa02a65c48635 100644 --- a/tests/tsim/inc/simInt.h +++ b/tests/tsim/inc/simInt.h @@ -143,7 +143,7 @@ typedef struct _script_t { char *optionBuffer; SCmdLine *lines; // command list SVariable variables[MAX_VAR_LEN]; - pthread_t bgPid; + TdThread bgPid; char auth[128]; struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; } SScript; diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 1857e217531a0fcef9f81b8dd258f1e3a4f9a3eb..855705e904060f1646799b54d0f670da2fb36dd7 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -293,7 +293,7 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { script->bgScripts[script->bgScriptLen++] = newScript; simInfo("script:%s, start to execute in background,", newScript->fileName); - if (pthread_create(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { + if (taosThreadCreate(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); return false; } else { diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 7a1950611eafd3c84cc7c2a34ecc17a473ba37c3..a0721941e34afe398146165f71b0d9c6141792b6 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -156,17 +156,17 @@ SScript *simBuildScriptObj(char *fileName) { if (cmdLine[i].jump == 0) cmdLine[i].jump = numOfLines; } - SScript *script = malloc(sizeof(SScript)); + SScript *script = taosMemoryMalloc(sizeof(SScript)); memset(script, 0, sizeof(SScript)); script->type = SIM_SCRIPT_TYPE_MAIN; script->numOfLines = numOfLines; tstrncpy(script->fileName, fileName, sizeof(script->fileName)); - script->optionBuffer = malloc(optionOffset); + script->optionBuffer = taosMemoryMalloc(optionOffset); memcpy(script->optionBuffer, optionBuffer, optionOffset); - script->lines = malloc(sizeof(SCmdLine) * numOfLines); + script->lines = taosMemoryMalloc(sizeof(SCmdLine) * numOfLines); memcpy(script->lines, cmdLine, sizeof(SCmdLine) * numOfLines); return script; @@ -239,7 +239,7 @@ SScript *simParseScript(char *fileName) { return NULL; } } - if(buffer != NULL) free(buffer); + if(buffer != NULL) taosMemoryFree(buffer); taosCloseFile(&pFile); script = simBuildScriptObj(fileName); diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index bfbfe2b0df8f0d1d680f593f400e4d78df268e35..eb5fb682642dce2601167f671370d9a2320c3b8b 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -55,21 +55,21 @@ void simFreeScript(SScript *script) { simDebug("script:%s, is background script, set stop flag", bgScript->fileName); bgScript->killed = true; if (taosCheckPthreadValid(bgScript->bgPid)) { - pthread_join(bgScript->bgPid, NULL); + taosThreadJoin(bgScript->bgPid, NULL); } simDebug("script:%s, background thread joined", bgScript->fileName); taos_close(bgScript->taos); - tfree(bgScript->lines); - tfree(bgScript->optionBuffer); - tfree(bgScript); + taosMemoryFreeClear(bgScript->lines); + taosMemoryFreeClear(bgScript->optionBuffer); + taosMemoryFreeClear(bgScript); } simDebug("script:%s, is cleaned", script->fileName); taos_close(script->taos); - tfree(script->lines); - tfree(script->optionBuffer); - tfree(script); + taosMemoryFreeClear(script->lines); + taosMemoryFreeClear(script->optionBuffer); + taosMemoryFreeClear(script); } } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index dae0a1c84023a649cfd14b9a188db46537e64882..f064833691a36e9ef3512d1648ea9a5934ab78e9 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -4,6 +4,7 @@ IF (TD_TAOS_TOOLS) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/common) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/util) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/os) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/libs/transport) ADD_SUBDIRECTORY(taos-tools) ENDIF () diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index b351675e4784a2ce31f321d8e224dc802fef1399..284693795ee471ad2d631758970c3033dc8e0c6c 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -4,9 +4,7 @@ add_executable(shell ${SHELL_SRC}) target_link_libraries( shell PUBLIC taos - PUBLIC util - PUBLIC common - PUBLIC os + PRIVATE os common transport util ) target_include_directories( shell diff --git a/tools/shell/inc/syncMsg.h b/tools/shell/inc/syncMsg.h new file mode 100644 index 0000000000000000000000000000000000000000..85ac9c78affa5282d5ca703caffc1bc5c24461bb --- /dev/null +++ b/tools/shell/inc/syncMsg.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_MSG_H +#define TDENGINE_SYNC_MSG_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "tsync.h" + +typedef enum { + TAOS_SMSG_START = 0, + TAOS_SMSG_SYNC_DATA = 1, + TAOS_SMSG_SYNC_DATA_RSP = 2, + TAOS_SMSG_SYNC_FWD = 3, + TAOS_SMSG_SYNC_FWD_RSP = 4, + TAOS_SMSG_SYNC_REQ = 5, + TAOS_SMSG_SYNC_REQ_RSP = 6, + TAOS_SMSG_SYNC_MUST = 7, + TAOS_SMSG_SYNC_MUST_RSP = 8, + TAOS_SMSG_STATUS = 9, + TAOS_SMSG_STATUS_RSP = 10, + TAOS_SMSG_SETUP = 11, + TAOS_SMSG_SETUP_RSP = 12, + TAOS_SMSG_SYNC_FILE = 13, + TAOS_SMSG_SYNC_FILE_RSP = 14, + TAOS_SMSG_TEST = 15, + TAOS_SMSG_END = 16 +} ESyncMsgType; + +typedef enum { + SYNC_STATUS_BROADCAST, + SYNC_STATUS_BROADCAST_RSP, + SYNC_STATUS_SETUP_CONN, + SYNC_STATUS_SETUP_CONN_RSP, + SYNC_STATUS_EXCHANGE_DATA, + SYNC_STATUS_EXCHANGE_DATA_RSP, + SYNC_STATUS_CHECK_ROLE, + SYNC_STATUS_CHECK_ROLE_RSP +} ESyncStatusType; + +#pragma pack(push, 1) + +typedef struct { + int8_t type; // msg type + int8_t protocol; // protocol version + uint16_t signature; // fixed value + int32_t code; // + int32_t cId; // cluster Id + int32_t vgId; // vg ID + int32_t len; // content length, does not include head + uint32_t cksum; +} SSyncHead; + +typedef struct { + SSyncHead head; + uint16_t port; + uint16_t tranId; + int32_t sourceId; // only for arbitrator + char fqdn[TSDB_FQDN_LEN]; +} SSyncMsg; + +typedef struct { + SSyncHead head; + int8_t sync; + int8_t reserved; + uint16_t tranId; + int8_t reserverd[4]; +} SSyncRsp; + +typedef struct { + int8_t role; + uint64_t version; +} SPeerStatus; + +typedef struct { + SSyncHead head; + int8_t role; + int8_t ack; + int8_t type; + int8_t reserved[3]; + uint16_t tranId; + uint64_t version; + SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA]; +} SPeersStatus; + +typedef struct { + SSyncHead head; + uint64_t fversion; +} SFileVersion; + +typedef struct { + SSyncHead head; + int8_t ack; +} SFileAck; + +typedef struct { + SSyncHead head; + uint64_t version; + int32_t code; +} SFwdRsp; + +#pragma pack(pop) + +#define SYNC_PROTOCOL_VERSION 1 +#define SYNC_SIGNATURE ((uint16_t)(0xCDEF)) + +extern char *statusType[]; + +uint16_t syncGenTranId(); +int32_t syncCheckHead(SSyncHead *pHead); + +void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len); +void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code); +void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); + +void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); +void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_VNODEPEER_H diff --git a/tools/shell/inc/tsync.h b/tools/shell/inc/tsync.h new file mode 100644 index 0000000000000000000000000000000000000000..d1b68e3f5a1e27b63dd08a4d1d4862c7b1e68179 --- /dev/null +++ b/tools/shell/inc/tsync.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_H +#define TDENGINE_SYNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TAOS_SYNC_MAX_REPLICA 5 +#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF + +typedef enum { + TAOS_SYNC_ROLE_OFFLINE = 0, + TAOS_SYNC_ROLE_UNSYNCED = 1, + TAOS_SYNC_ROLE_SYNCING = 2, + TAOS_SYNC_ROLE_SLAVE = 3, + TAOS_SYNC_ROLE_MASTER = 4 +} ESyncRole; + +typedef enum { + TAOS_SYNC_STATUS_INIT = 0, + TAOS_SYNC_STATUS_START = 1, + TAOS_SYNC_STATUS_FILE = 2, + TAOS_SYNC_STATUS_CACHE = 3 +} ESyncStatus; + +typedef struct { + uint32_t nodeId; // node ID assigned by TDengine + uint16_t nodePort; // node sync Port + char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN +} SNodeInfo; + +typedef struct { + int8_t quorum; // number of confirms required, >=1 + int8_t replica; // number of replications, >=1 + SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; +} SSyncCfg; + +typedef struct { + int32_t selfIndex; + uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; + int32_t role[TAOS_SYNC_MAX_REPLICA]; +} SNodesRole; + +// get the wal file from index or after +// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file +typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); + +// when a forward pkt is received, call this to handle data +typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); + +// when forward is confirmed by peer, master call this API to notify app +typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); + +// when role is changed, call this to notify app +typedef void (*FNotifyRole)(int32_t vgId, int8_t role); + +// if a number of retrieving data failed, call this to start flow control +typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); + +// when data file is synced successfully, notity app +typedef void (*FStartSyncFile)(int32_t vgId); +typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); + +// get file version +typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); + +typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); +typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); + +typedef struct { + int32_t vgId; // vgroup ID + uint64_t version; // initial version + SSyncCfg syncCfg; // configuration from mgmt + char path[TSDB_FILENAME_LEN]; // path to the file + void * pTsdb; + FGetWalInfo getWalInfoFp; + FWriteToCache writeToCacheFp; + FConfirmForward confirmForward; + FNotifyRole notifyRoleFp; + FNotifyFlowCtrl notifyFlowCtrlFp; + FStartSyncFile startSyncFileFp; + FStopSyncFile stopSyncFileFp; + FGetVersion getVersionFp; + FSendFile sendFileFp; + FRecvFile recvFileFp; +} SSyncInfo; + +typedef void *tsync_h; + +int32_t syncInit(); +void syncCleanUp(); + +int64_t syncStart(const SSyncInfo *); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg *); +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force); +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force); +void syncRecover(int64_t rid); // recover from other nodes: +int32_t syncGetNodesRole(int64_t rid, SNodesRole *); + +extern char *syncRole[]; + +//global configurable parameters +extern int32_t sDebugFlag; +extern char tsArbitrator[]; +extern uint16_t tsSyncPort; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SYNC_H diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 33d25b67464e52cb6808f6b845963cf65ee49aac..dc18ecd3f8d877a4bb158cffee9a6cda02edd221 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -31,7 +31,7 @@ static int32_t checkedNum = 0; static int32_t errorNum = 0; typedef struct { - pthread_t threadID; + TdThread threadID; int threadIndex; int totalThreads; void * taos; @@ -72,7 +72,7 @@ static int32_t shellShowTables(TAOS *con, char *db) { int32_t tbIndex = tbNum++; if (tbMallocNum < tbNum) { tbMallocNum = (tbMallocNum * 2 + 1); - char** tbNames1 = realloc(tbNames, tbMallocNum * sizeof(char *)); + char** tbNames1 = taosMemoryRealloc(tbNames, tbMallocNum * sizeof(char *)); if (tbNames1 == NULL) { fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum); code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -81,7 +81,7 @@ static int32_t shellShowTables(TAOS *con, char *db) { tbNames = tbNames1; } - tbNames[tbIndex] = malloc(TSDB_TABLE_NAME_LEN); + tbNames[tbIndex] = taosMemoryMalloc(TSDB_TABLE_NAME_LEN); strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN); if (tbIndex % 100000 == 0 && tbIndex != 0) { fprintf(stdout, "%d tablenames fetched\n", tbIndex); @@ -97,9 +97,9 @@ static int32_t shellShowTables(TAOS *con, char *db) { static void shellFreeTbnames() { for (int32_t i = 0; i < tbNum; ++i) { - free(tbNames[i]); + taosMemoryFree(tbNames[i]); } - free(tbNames); + taosMemoryFree(tbNames); } static void *shellCheckThreadFp(void *arg) { @@ -152,8 +152,8 @@ static void *shellCheckThreadFp(void *arg) { } static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { - pthread_attr_t thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); + TdThreadAttr thattr; + ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; pThread->threadIndex = t; @@ -161,23 +161,23 @@ static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { pThread->taos = con; pThread->db = _args->database; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { + if (taosThreadCreate(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); exit(0); } } for (int t = 0; t < _args->threadNum; ++t) { - pthread_join(threadObj[t].threadID, NULL); + taosThreadJoin(threadObj[t].threadID, NULL); } for (int t = 0; t < _args->threadNum; ++t) { taos_close(threadObj[t].taos); } - free(threadObj); + taosMemoryFree(threadObj); } void shellCheck(TAOS *con, SShellArguments *_args) { diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index d7a976d52cc6179726cf80645b28f5bb30bc6fac..93335776bad446d4ad9575efa838ed1723421901 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -206,8 +206,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { char utf8_array[10] = "\0"; Command cmd; memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); showOnScreen(&cmd); // Read input. @@ -252,8 +252,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - tfree(cmd.buffer); - tfree(cmd.command); + taosMemoryFreeClear(cmd.buffer); + taosMemoryFreeClear(cmd.command); return 0; } else { updateBuffer(&cmd); @@ -365,9 +365,9 @@ void *shellLoopQuery(void *arg) { setThreadName("shellLoopQuery"); - pthread_cleanup_push(cleanup_handler, NULL); + taosThreadCleanupPush(cleanup_handler, NULL); - char *command = malloc(MAX_COMMAND_SIZE); + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); if (command == NULL){ tscError("failed to malloc command"); return NULL; @@ -386,10 +386,10 @@ void *shellLoopQuery(void *arg) { resetTerminalMode(); } while (shellRunCommand(con, command) == 0); - tfree(command); + taosMemoryFreeClear(command); exitShell(); - pthread_cleanup_pop(1); + taosThreadCleanupPop(1); return NULL; } @@ -429,7 +429,7 @@ void showOnScreen(Command *cmd) { int size = 0; // Print out the command. - char *total_string = malloc(MAX_COMMAND_SIZE); + char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); memset(total_string, '\0', MAX_COMMAND_SIZE); if (strcmp(cmd->buffer, "") == 0) { sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); @@ -461,7 +461,7 @@ void showOnScreen(Command *cmd) { str = total_string + size; } - free(total_string); + taosMemoryFree(total_string); /* for (int i = 0; i < size; i++){ */ /* char c = total_string[i]; */ /* if (k % w.ws_col == 0) { */ diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index b42f77e87e0a90e152d1a8b937ab8a27e6a5ec20..130c72a20bbdb831fb1fe3fa8d62ca93f5aa45dd 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -28,7 +28,7 @@ static int32_t shellSQLFileNum = 0; static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0}; typedef struct { - pthread_t threadID; + TdThread threadID; int threadIndex; int totalThreads; void *taos; @@ -97,9 +97,9 @@ static void shellCheckTablesSQLFile(const char *directoryName) static void shellMallocSQLFiles() { - shellSQLFiles = (char**)calloc(shellSQLFileNum, sizeof(char*)); + shellSQLFiles = (char**)taosMemoryCalloc(shellSQLFileNum, sizeof(char*)); for (int i = 0; i < shellSQLFileNum; i++) { - shellSQLFiles[i] = calloc(1, TSDB_FILENAME_LEN); + shellSQLFiles[i] = taosMemoryCalloc(1, TSDB_FILENAME_LEN); } } @@ -130,20 +130,20 @@ static void shellGetDirectoryFileList(char *inputDir) static void shellSourceFile(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char * cmd = malloc(tsMaxSQLStringLen); + char * cmd = taosMemoryMalloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); - free(cmd); + taosMemoryFree(cmd); return; } char *fname = full_path.we_wordv[0]; if (fname == NULL) { fprintf(stderr, "ERROR: invalid filename\n"); - free(cmd); + taosMemoryFree(cmd); return; } @@ -152,7 +152,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -160,7 +160,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not readable\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } */ @@ -170,7 +170,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -209,8 +209,8 @@ static void shellSourceFile(TAOS *con, char *fptr) { cmd_len = 0; } - free(cmd); - if(line != NULL) free(line); + taosMemoryFree(cmd); + if(line != NULL) taosMemoryFree(line); wordfree(&full_path); taosCloseFile(&pFile); } @@ -232,8 +232,8 @@ void* shellImportThreadFp(void *arg) static void shellRunImportThreads(SShellArguments* _args) { - pthread_attr_t thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); + TdThreadAttr thattr; + ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; pThread->threadIndex = t; @@ -244,23 +244,23 @@ static void shellRunImportThreads(SShellArguments* _args) exit(0); } - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) { + if (taosThreadCreate(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) { fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); exit(0); } } for (int t = 0; t < _args->threadNum; ++t) { - pthread_join(threadObj[t].threadID, NULL); + taosThreadJoin(threadObj[t].threadID, NULL); } for (int t = 0; t < _args->threadNum; ++t) { taos_close(threadObj[t].taos); } - free(threadObj); + taosMemoryFree(threadObj); } void source_dir(TAOS* con, SShellArguments* _args) { diff --git a/tools/shell/src/backup/shellWindows.c b/tools/shell/src/backup/shellWindows.c index 1244c7b06022a53380c1a84304a4a2cdb76a17d2..92ac7fd7213b13b34d94714b32216b1fb0435477 100644 --- a/tools/shell/src/backup/shellWindows.c +++ b/tools/shell/src/backup/shellWindows.c @@ -239,7 +239,7 @@ void updateBuffer(Command *cmd) { } int isReadyGo(Command *cmd) { - char *total = malloc(MAX_COMMAND_SIZE); + char *total = taosMemoryMalloc(MAX_COMMAND_SIZE); memset(total, 0, MAX_COMMAND_SIZE); sprintf(total, "%s%s", cmd->buffer, cmd->command); @@ -247,11 +247,11 @@ int isReadyGo(Command *cmd) { "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" "\\s*clear\\s*$)"; if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { - free(total); + taosMemoryFree(total); return 1; } - free(total); + taosMemoryFree(total); return 0; } @@ -268,8 +268,8 @@ void insertChar(Command *cmd, char c) { int32_t shellReadCommand(TAOS *con, char command[]) { Command cmd; memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); // Read input. char c; @@ -281,9 +281,9 @@ int32_t shellReadCommand(TAOS *con, char command[]) { case '\r': if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - free(cmd.buffer); + taosMemoryFree(cmd.buffer); cmd.buffer = NULL; - free(cmd.command); + taosMemoryFree(cmd.command); cmd.command = NULL; return 0; } else { @@ -301,7 +301,7 @@ int32_t shellReadCommand(TAOS *con, char command[]) { void *shellLoopQuery(void *arg) { TAOS *con = (TAOS *)arg; - char *command = malloc(MAX_COMMAND_SIZE); + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); if (command == NULL) return NULL; int32_t err = 0; diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index fd993998b823411c666194d9f2bb9c6390baa2c4..7cbadfaf5b015a3f6ef039df1370c9381238ef99 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -230,7 +230,7 @@ void updateBuffer(Command *cmd) { int isReadyGo(Command *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - char *total = (char *)calloc(1, MAX_COMMAND_SIZE); + char *total = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); memset(cmd->command + cmd->commandSize, 0, MAX_COMMAND_SIZE - cmd->commandSize); sprintf(total, "%s%s", cmd->buffer, cmd->command); @@ -238,20 +238,20 @@ int isReadyGo(Command *cmd) { "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" "\\s*clear\\s*$)"; if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { - free(total); + taosMemoryFree(total); return 1; } - free(total); + taosMemoryFree(total); return 0; } void getMbSizeInfo(const char *str, int *size, int *width) { - TdWchar *wc = (TdWchar *)calloc(sizeof(TdWchar), MAX_COMMAND_SIZE); + TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), MAX_COMMAND_SIZE); *size = strlen(str); taosMbsToWchars(wc, str, MAX_COMMAND_SIZE); *width = taosWcharsWidth(wc, MAX_COMMAND_SIZE); - free(wc); + taosMemoryFree(wc); } void resetCommand(Command *cmd, const char s[]) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1b35afb57d231c07181499fa43c66cbad137e72c..47d47f023df21d8114f2c736350c0a6ce5ed1767 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -210,7 +210,7 @@ int32_t shellRunCommand(TAOS *con, char *command) { history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL || strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) { if (history.hist[history.hend] != NULL) { - tfree(history.hist[history.hend]); + taosMemoryFreeClear(history.hist[history.hend]); } history.hist[history.hend] = strdup(command); @@ -358,6 +358,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } else { printf("Query interrupted (%s), %d row(s) in set (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); } + taos_free_result(pSql); } else { int num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); @@ -925,7 +926,7 @@ void read_history() { } } - if(line != NULL) free(line); + if(line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); } @@ -945,7 +946,7 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { taosFprintfFile(pFile, "%s\n", history.hist[i]); - tfree(history.hist[i]); + taosMemoryFreeClear(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } @@ -968,13 +969,13 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); + char *cmd = taosMemoryCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); - free(cmd); + taosMemoryFree(cmd); return; } @@ -985,7 +986,7 @@ void source_file(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } */ @@ -995,7 +996,7 @@ void source_file(TAOS *con, char *fptr) { if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -1021,8 +1022,8 @@ void source_file(TAOS *con, char *fptr) { cmd_len = 0; } - free(cmd); - if(line != NULL) free(line); + taosMemoryFree(cmd); + if(line != NULL) taosMemoryFree(line); wordfree(&full_path); taosCloseFile(&pFile); } diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c deleted file mode 100644 index 6da05f28dfbf1c74ea6fa4aeb111053c89e80160..0000000000000000000000000000000000000000 --- a/tools/shell/src/shellLinux.c +++ /dev/null @@ -1,529 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define __USE_XOPEN -#include "os.h" -#include "tglobal.h" -#include "shell.h" -#include "shellCommand.h" -#include "tbase64.h" -#include "tlog.h" -#include "version.h" - -#include -#include -#include - -#define OPT_ABORT 1 /* �Cabort */ - -int indicator = 1; -struct termios oldtio; - -void insertChar(Command *cmd, char *c, int size); -const char *argp_program_version = version; -const char *argp_program_bug_address = ""; -static char doc[] = ""; -static char args_doc[] = ""; -static struct argp_option options[] = { - {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, - {"password", 'p', 0, 0, "The password to use when connecting to the server."}, - {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, - {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, - {"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."}, - {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, - {"dump-config", 'C', 0, 0, "Dump configuration."}, - {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, - {"raw-time", 'r', 0, 0, "Output time as uint64_t."}, - {"file", 'f', "FILE", 0, "Script to run without enter the shell."}, - {"directory", 'D', "DIRECTORY", 0, "Use multi-thread to import all SQL files in the directory separately."}, - {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, - {"check", 'k', "CHECK", 0, "Check tables."}, - {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, - {"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speen|fqdn."}, - {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, - {"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."}, - {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, - {0}}; - -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - /* Get the input argument from argp_parse, which we - know is a pointer to our arguments structure. */ - SShellArguments *arguments = state->input; - wordexp_t full_path; - - switch (key) { - case 'h': - arguments->host = arg; - break; - case 'p': - break; - case 'P': - if (arg) { - arguments->port = atoi(arg); - } else { - fprintf(stderr, "Invalid port\n"); - return -1; - } - - break; - case 'z': - arguments->timezone = arg; - break; - case 'u': - arguments->user = arg; - break; - case 'A': - arguments->auth = arg; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) { - fprintf(stderr, "config file path: %s overflow max len %d\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1); - wordfree(&full_path); - return -1; - } - tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'C': - arguments->dump_config = true; - break; - case 's': - arguments->commands = arg; - break; - case 'r': - arguments->is_raw_time = true; - break; - case 'f': - if ((0 == strlen(arg)) || (wordexp(arg, &full_path, 0) != 0)) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'D': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'T': - if (arg) { - arguments->threadNum = atoi(arg); - } else { - fprintf(stderr, "Invalid number of threads\n"); - return -1; - } - break; - case 'k': - arguments->check = atoi(arg); - break; - case 'd': - arguments->database = arg; - break; - case 'n': - arguments->netTestRole = arg; - break; - case 'l': - if (arg) { - arguments->pktLen = atoi(arg); - } else { - fprintf(stderr, "Invalid packet length\n"); - return -1; - } - break; - case 'N': - if (arg) { - arguments->pktNum = atoi(arg); - } else { - fprintf(stderr, "Invalid packet number\n"); - return -1; - } - break; - case 'S': - arguments->pktType = arg; - break; - case OPT_ABORT: - arguments->abort = 1; - break; - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - -/* Our argp parser. */ -static struct argp argp = {options, parse_opt, args_doc, doc}; - -char LINUXCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[SHELL_MAX_PASSWORD_LEN]; - -static void parse_args( - int argc, char *argv[], SShellArguments *arguments) { - for (int i = 1; i < argc; i++) { - if ((strncmp(argv[i], "-p", 2) == 0) - || (strncmp(argv[i], "--password", 10) == 0)) { - printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); - if ((strlen(argv[i]) == 2) - || (strncmp(argv[i], "--password", 10) == 0)) { - printf("Enter password: "); - taosSetConsoleEcho(false); - if (scanf("%20s", g_password) > 1) { - fprintf(stderr, "password reading error\n"); - } - taosSetConsoleEcho(true); - if (EOF == getchar()) { - fprintf(stderr, "getchar() return EOF\n"); - } - } else { - tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); - strcpy(argv[i], "-p"); - } - arguments->password = g_password; - arguments->is_use_passwd = true; - } - } -} - -void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { - static char verType[32] = {0}; - sprintf(verType, "version: %s\n", version); - - argp_program_version = verType; - - if (argc > 1) { - parse_args(argc, argv, arguments); - } - - argp_parse(&argp, argc, argv, 0, 0, arguments); - if (arguments->abort) { - #ifndef _ALPINE - #if 0 - error(10, 0, "ABORTED"); - #endif - #else - abort(); - #endif - } -} - -int32_t shellReadCommand(TAOS *con, char *command) { - unsigned hist_counter = history.hend; - char utf8_array[10] = "\0"; - Command cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); - showOnScreen(&cmd); - - // Read input. - char c; - while (1) { - c = (char)getchar(); // getchar() return an 'int' value - - if (c == EOF) { - return c; - } - - if (c < 0) { // For UTF-8 - int count = countPrefixOnes(c); - utf8_array[0] = c; - for (int k = 1; k < count; k++) { - c = (char)getchar(); - utf8_array[k] = c; - } - insertChar(&cmd, utf8_array, count); - } else if (c < '\033') { - // Ctrl keys. TODO: Implement ctrl combinations - switch (c) { - case 1: // ctrl A - positionCursorHome(&cmd); - break; - case 3: - printf("\n"); - resetCommand(&cmd, ""); - kill(0, SIGINT); - break; - case 4: // EOF or Ctrl+D - printf("\n"); - taos_close(con); - // write the history - write_history(); - exitShell(); - break; - case 5: // ctrl E - positionCursorEnd(&cmd); - break; - case 8: - backspaceChar(&cmd); - break; - case '\n': - case '\r': - printf("\n"); - if (isReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - tfree(cmd.buffer); - tfree(cmd.command); - return 0; - } else { - updateBuffer(&cmd); - } - break; - case 11: // Ctrl + K; - clearLineAfter(&cmd); - break; - case 12: // Ctrl + L; - system("clear"); - showOnScreen(&cmd); - break; - case 21: // Ctrl + U; - clearLineBefore(&cmd); - break; - } - } else if (c == '\033') { - c = (char)getchar(); - switch (c) { - case '[': - c = (char)getchar(); - switch (c) { - case 'A': // Up arrow - if (hist_counter != history.hstart) { - hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE; - resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]); - } - break; - case 'B': // Down arrow - if (hist_counter != history.hend) { - int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE; - - if (next_hist != history.hend) { - resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]); - } else { - resetCommand(&cmd, ""); - } - hist_counter = next_hist; - } - break; - case 'C': // Right arrow - moveCursorRight(&cmd); - break; - case 'D': // Left arrow - moveCursorLeft(&cmd); - break; - case '1': - if ((c = (char)getchar()) == '~') { - // Home key - positionCursorHome(&cmd); - } - break; - case '2': - if ((c = (char)getchar()) == '~') { - // Insert key - } - break; - case '3': - if ((c = (char)getchar()) == '~') { - // Delete key - deleteChar(&cmd); - } - break; - case '4': - if ((c = (char)getchar()) == '~') { - // End key - positionCursorEnd(&cmd); - } - break; - case '5': - if ((c = (char)getchar()) == '~') { - // Page up key - } - break; - case '6': - if ((c = (char)getchar()) == '~') { - // Page down key - } - break; - case 72: - // Home key - positionCursorHome(&cmd); - break; - case 70: - // End key - positionCursorEnd(&cmd); - break; - } - break; - } - } else if (c == 0x7f) { - // press delete key - backspaceChar(&cmd); - } else { - insertChar(&cmd, &c, 1); - } - } - - return 0; -} - -void *shellLoopQuery(void *arg) { - if (indicator) { - getOldTerminalMode(); - indicator = 0; - } - - TAOS *con = (TAOS *)arg; - - setThreadName("shellLoopQuery"); - - pthread_cleanup_push(cleanup_handler, NULL); - - char *command = malloc(MAX_COMMAND_SIZE); - if (command == NULL){ - uError("failed to malloc command"); - return NULL; - } - - int32_t err = 0; - - do { - // Read command from shell. - memset(command, 0, MAX_COMMAND_SIZE); - setTerminalMode(); - err = shellReadCommand(con, command); - if (err) { - break; - } - resetTerminalMode(); - } while (shellRunCommand(con, command) == 0); - - tfree(command); - exitShell(); - - pthread_cleanup_pop(1); - - return NULL; -} - -void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } - -void clearScreen(int ecmd_pos, int cursor_pos) { - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - positionCursor(cursor_y, LEFT); - positionCursor(command_x - cursor_x, DOWN); - fprintf(stdout, "\033[2K"); - for (int i = 0; i < command_x; i++) { - positionCursor(1, UP); - fprintf(stdout, "\033[2K"); - } - fflush(stdout); -} - -void showOnScreen(Command *cmd) { - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - //fprintf(stderr, "No stream device\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - TdWchar wc; - int size = 0; - - // Print out the command. - char *total_string = malloc(MAX_COMMAND_SIZE); - memset(total_string, '\0', MAX_COMMAND_SIZE); - if (strcmp(cmd->buffer, "") == 0) { - sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); - } else { - sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command); - } - - int remain_column = w.ws_col; - /* size = cmd->commandSize + prompt_size; */ - for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); - if (ret < 0) break; - size += ret; - /* assert(size >= 0); */ - int width = taosWcharWidth(wc); - if (remain_column > width) { - printf("%lc", wc); - remain_column -= width; - } else { - if (remain_column == width) { - printf("%lc\n\r", wc); - remain_column = w.ws_col; - } else { - printf("\n\r%lc", wc); - remain_column = w.ws_col - width; - } - } - - str = total_string + size; - } - - free(total_string); - /* for (int i = 0; i < size; i++){ */ - /* char c = total_string[i]; */ - /* if (k % w.ws_col == 0) { */ - /* printf("%c\n\r", c); */ - /* } */ - /* else { */ - /* printf("%c", c); */ - /* } */ - /* k += 1; */ - /* } */ - - // Position the cursor - int cursor_pos = cmd->screenOffset + prompt_size; - int ecmd_pos = cmd->endOffset + prompt_size; - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - // int cursor_y = cursor % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - int command_y = ecmd_pos % w.ws_col; - // int command_y = (command.size() + prompt_size) % w.ws_col; - positionCursor(command_y, LEFT); - positionCursor(command_x, UP); - positionCursor(cursor_x, DOWN); - positionCursor(cursor_y, RIGHT); - fflush(stdout); -} - -void cleanup_handler(void *arg) { resetTerminalMode(); } - -void exitShell() { - /*int32_t ret =*/ resetTerminalMode(); - taos_cleanup(); - exit(EXIT_SUCCESS); -} diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 2832855517e2529d511a39adcf749b87a6c7d1d3..70563c79e697eee201798926f69f1af68244af20 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -13,13 +13,528 @@ * along with this program. If not, see . */ +#define __USE_XOPEN #include "os.h" #include "shell.h" #include "tglobal.h" +#include "tconfig.h" +#include "shellCommand.h" +#include "tbase64.h" +#include "tlog.h" +#include "version.h" -pthread_t pid; +#include +#include +#include + +#define OPT_ABORT 1 /* abort */ + + +int indicator = 1; + +void insertChar(Command *cmd, char *c, int size); +void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, + int32_t pkgNum, char *pkgType); +const char *argp_program_version = version; +const char *argp_program_bug_address = ""; +static char doc[] = ""; +static char args_doc[] = ""; + +TdThread pid; static tsem_t cancelSem; +static struct argp_option options[] = { + {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, + {"password", 'p', 0, 0, "The password to use when connecting to the server."}, + {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, + {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, + {"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."}, + {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, + {"dump-config", 'C', 0, 0, "Dump configuration."}, + {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, + {"raw-time", 'r', 0, 0, "Output time as uint64_t."}, + {"file", 'f', "FILE", 0, "Script to run without enter the shell."}, + {"directory", 'D', "DIRECTORY", 0, "Use multi-thread to import all SQL files in the directory separately."}, + {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, + {"check", 'k', "CHECK", 0, "Check tables."}, + {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, + {"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."}, + {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn."}, + {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, + {"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."}, +// Shuduo: 3.0 does not support UDP any more +// {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, + {0}}; + +static error_t parse_opt(int key, char *arg, struct argp_state *state) { + /* Get the input argument from argp_parse, which we + know is a pointer to our arguments structure. */ + SShellArguments *arguments = state->input; + wordexp_t full_path; + + switch (key) { + case 'h': + arguments->host = arg; + break; + case 'p': + break; + case 'P': + if (arg) { + arguments->port = atoi(arg); + } else { + fprintf(stderr, "Invalid port\n"); + return -1; + } + + break; + case 'z': + arguments->timezone = arg; + break; + case 'u': + arguments->user = arg; + break; + case 'A': + arguments->auth = arg; + break; + case 'c': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid path %s\n", arg); + return -1; + } + if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) { + fprintf(stderr, "config file path: %s overflow max len %d\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1); + wordfree(&full_path); + return -1; + } + tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); + wordfree(&full_path); + break; + case 'C': + arguments->dump_config = true; + break; + case 's': + arguments->commands = arg; + break; + case 'r': + arguments->is_raw_time = true; + break; + case 'f': + if ((0 == strlen(arg)) || (wordexp(arg, &full_path, 0) != 0)) { + fprintf(stderr, "Invalid path %s\n", arg); + return -1; + } + tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN); + wordfree(&full_path); + break; + case 'D': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid path %s\n", arg); + return -1; + } + tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN); + wordfree(&full_path); + break; + case 'T': + if (arg) { + arguments->threadNum = atoi(arg); + } else { + fprintf(stderr, "Invalid number of threads\n"); + return -1; + } + break; + case 'k': + arguments->check = atoi(arg); + break; + case 'd': + arguments->database = arg; + break; + case 'n': + arguments->netTestRole = arg; + break; + case 'l': + if (arg) { + arguments->pktLen = atoi(arg); + } else { + fprintf(stderr, "Invalid packet length\n"); + return -1; + } + break; + case 'N': + if (arg) { + arguments->pktNum = atoi(arg); + } else { + fprintf(stderr, "Invalid packet number\n"); + return -1; + } + break; + case 'S': + arguments->pktType = arg; + break; + case OPT_ABORT: + arguments->abort = 1; + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +/* Our argp parser. */ +static struct argp argp = {options, parse_opt, args_doc, doc}; + +char LINUXCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; +char g_password[SHELL_MAX_PASSWORD_LEN]; + +static void parse_args( + int argc, char *argv[], SShellArguments *arguments) { + for (int i = 1; i < argc; i++) { + if ((strncmp(argv[i], "-p", 2) == 0) + || (strncmp(argv[i], "--password", 10) == 0)) { + printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); + if ((strlen(argv[i]) == 2) + || (strncmp(argv[i], "--password", 10) == 0)) { + printf("Enter password: "); + taosSetConsoleEcho(false); + if (scanf("%20s", g_password) > 1) { + fprintf(stderr, "password reading error\n"); + } + taosSetConsoleEcho(true); + if (EOF == getchar()) { + fprintf(stderr, "getchar() return EOF\n"); + } + } else { + tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); + strcpy(argv[i], "-p"); + } + arguments->password = g_password; + arguments->is_use_passwd = true; + } + } +} + +void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { + static char verType[32] = {0}; + sprintf(verType, "version: %s\n", version); + + argp_program_version = verType; + + if (argc > 1) { + parse_args(argc, argv, arguments); + } + + argp_parse(&argp, argc, argv, 0, 0, arguments); + if (arguments->abort) { + #ifndef _ALPINE + #if 0 + error(10, 0, "ABORTED"); + #endif + #else + abort(); + #endif + } +} + +int32_t shellReadCommand(TAOS *con, char *command) { + unsigned hist_counter = history.hend; + char utf8_array[10] = "\0"; + Command cmd; + memset(&cmd, 0, sizeof(cmd)); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + showOnScreen(&cmd); + + // Read input. + char c; + while (1) { + c = (char)getchar(); // getchar() return an 'int' value + + if (c == EOF) { + return c; + } + + if (c < 0) { // For UTF-8 + int count = countPrefixOnes(c); + utf8_array[0] = c; + for (int k = 1; k < count; k++) { + c = (char)getchar(); + utf8_array[k] = c; + } + insertChar(&cmd, utf8_array, count); + } else if (c < '\033') { + // Ctrl keys. TODO: Implement ctrl combinations + switch (c) { + case 1: // ctrl A + positionCursorHome(&cmd); + break; + case 3: + printf("\n"); + resetCommand(&cmd, ""); + kill(0, SIGINT); + break; + case 4: // EOF or Ctrl+D + printf("\n"); + taos_close(con); + // write the history + write_history(); + exitShell(); + break; + case 5: // ctrl E + positionCursorEnd(&cmd); + break; + case 8: + backspaceChar(&cmd); + break; + case '\n': + case '\r': + printf("\n"); + if (isReadyGo(&cmd)) { + sprintf(command, "%s%s", cmd.buffer, cmd.command); + taosMemoryFreeClear(cmd.buffer); + taosMemoryFreeClear(cmd.command); + return 0; + } else { + updateBuffer(&cmd); + } + break; + case 11: // Ctrl + K; + clearLineAfter(&cmd); + break; + case 12: // Ctrl + L; + system("clear"); + showOnScreen(&cmd); + break; + case 21: // Ctrl + U; + clearLineBefore(&cmd); + break; + } + } else if (c == '\033') { + c = (char)getchar(); + switch (c) { + case '[': + c = (char)getchar(); + switch (c) { + case 'A': // Up arrow + if (hist_counter != history.hstart) { + hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE; + resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]); + } + break; + case 'B': // Down arrow + if (hist_counter != history.hend) { + int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE; + + if (next_hist != history.hend) { + resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]); + } else { + resetCommand(&cmd, ""); + } + hist_counter = next_hist; + } + break; + case 'C': // Right arrow + moveCursorRight(&cmd); + break; + case 'D': // Left arrow + moveCursorLeft(&cmd); + break; + case '1': + if ((c = (char)getchar()) == '~') { + // Home key + positionCursorHome(&cmd); + } + break; + case '2': + if ((c = (char)getchar()) == '~') { + // Insert key + } + break; + case '3': + if ((c = (char)getchar()) == '~') { + // Delete key + deleteChar(&cmd); + } + break; + case '4': + if ((c = (char)getchar()) == '~') { + // End key + positionCursorEnd(&cmd); + } + break; + case '5': + if ((c = (char)getchar()) == '~') { + // Page up key + } + break; + case '6': + if ((c = (char)getchar()) == '~') { + // Page down key + } + break; + case 72: + // Home key + positionCursorHome(&cmd); + break; + case 70: + // End key + positionCursorEnd(&cmd); + break; + } + break; + } + } else if (c == 0x7f) { + // press delete key + backspaceChar(&cmd); + } else { + insertChar(&cmd, &c, 1); + } + } + + return 0; +} + +void *shellLoopQuery(void *arg) { + if (indicator) { + getOldTerminalMode(); + indicator = 0; + } + + TAOS *con = (TAOS *)arg; + + setThreadName("shellLoopQuery"); + + taosThreadCleanupPush(cleanup_handler, NULL); + + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); + if (command == NULL){ + uError("failed to malloc command"); + return NULL; + } + + int32_t err = 0; + + do { + // Read command from shell. + memset(command, 0, MAX_COMMAND_SIZE); + setTerminalMode(); + err = shellReadCommand(con, command); + if (err) { + break; + } + resetTerminalMode(); + } while (shellRunCommand(con, command) == 0); + + taosMemoryFreeClear(command); + exitShell(); + + taosThreadCleanupPop(1); + + return NULL; +} + +void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } + +void clearScreen(int ecmd_pos, int cursor_pos) { + struct winsize w; + if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { + //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); + w.ws_col = 120; + w.ws_row = 30; + } + + int cursor_x = cursor_pos / w.ws_col; + int cursor_y = cursor_pos % w.ws_col; + int command_x = ecmd_pos / w.ws_col; + positionCursor(cursor_y, LEFT); + positionCursor(command_x - cursor_x, DOWN); + fprintf(stdout, "\033[2K"); + for (int i = 0; i < command_x; i++) { + positionCursor(1, UP); + fprintf(stdout, "\033[2K"); + } + fflush(stdout); +} + +void showOnScreen(Command *cmd) { + struct winsize w; + if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { + //fprintf(stderr, "No stream device\n"); + w.ws_col = 120; + w.ws_row = 30; + } + + TdWchar wc; + int size = 0; + + // Print out the command. + char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); + memset(total_string, '\0', MAX_COMMAND_SIZE); + if (strcmp(cmd->buffer, "") == 0) { + sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); + } else { + sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command); + } + + int remain_column = w.ws_col; + /* size = cmd->commandSize + prompt_size; */ + for (char *str = total_string; size < cmd->commandSize + prompt_size;) { + int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); + if (ret < 0) break; + size += ret; + /* assert(size >= 0); */ + int width = taosWcharWidth(wc); + if (remain_column > width) { + printf("%lc", wc); + remain_column -= width; + } else { + if (remain_column == width) { + printf("%lc\n\r", wc); + remain_column = w.ws_col; + } else { + printf("\n\r%lc", wc); + remain_column = w.ws_col - width; + } + } + + str = total_string + size; + } + + taosMemoryFree(total_string); + /* for (int i = 0; i < size; i++){ */ + /* char c = total_string[i]; */ + /* if (k % w.ws_col == 0) { */ + /* printf("%c\n\r", c); */ + /* } */ + /* else { */ + /* printf("%c", c); */ + /* } */ + /* k += 1; */ + /* } */ + + // Position the cursor + int cursor_pos = cmd->screenOffset + prompt_size; + int ecmd_pos = cmd->endOffset + prompt_size; + + int cursor_x = cursor_pos / w.ws_col; + int cursor_y = cursor_pos % w.ws_col; + // int cursor_y = cursor % w.ws_col; + int command_x = ecmd_pos / w.ws_col; + int command_y = ecmd_pos % w.ws_col; + // int command_y = (command.size() + prompt_size) % w.ws_col; + positionCursor(command_y, LEFT); + positionCursor(command_x, UP); + positionCursor(cursor_x, DOWN); + positionCursor(cursor_y, RIGHT); + fflush(stdout); +} + +void cleanup_handler(void *arg) { resetTerminalMode(); } + +void exitShell() { + /*int32_t ret =*/ resetTerminalMode(); + taos_cleanup(); + exit(EXIT_SUCCESS); +} void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); } @@ -105,29 +620,35 @@ int main(int argc, char *argv[]) { shellParseArgument(argc, argv, &args); -#if 0 if (args.dump_config) { - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); + taosInitCfg(configDir, NULL, NULL, NULL, 1); - if (taosReadGlobalCfg() ! =0) { - printf("TDengine read global config failed"); + SConfig *pCfg = taosGetCfg(); + if (NULL == pCfg) { + printf("TDengine read global config failed!\n"); exit(EXIT_FAILURE); } - - taosDumpGlobalCfg(); + cfgDumpCfg(pCfg, 0, 1); exit(0); } if (args.netTestRole && args.netTestRole[0] != 0) { - if (taos_init()) { + TAOS *con = NULL; + if (args.auth == NULL) { + con = taos_connect(args.host, args.user, args.password, args.database, args.port); + } else { + con = taos_connect_auth(args.host, args.user, args.auth, args.database, args.port); + } + +/* if (taos_init()) { printf("Failed to init taos"); exit(EXIT_FAILURE); } + */ taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); + taos_close(con); exit(0); } -#endif /* Initialize the shell */ TAOS *con = shellInit(&args); @@ -140,8 +661,8 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } - pthread_t spid; - pthread_create(&spid, NULL, cancelHandler, NULL); + TdThread spid; + taosThreadCreate(&spid, NULL, cancelHandler, NULL); /* Interrupt handler. */ taosSetSignal(SIGTERM, shellQueryInterruptHandler); @@ -154,7 +675,7 @@ int main(int argc, char *argv[]) { /* Loop to query the input. */ while (1) { - pthread_create(&pid, NULL, shellLoopQuery, con); - pthread_join(pid, NULL); + taosThreadCreate(&pid, NULL, shellLoopQuery, con); + taosThreadJoin(pid, NULL); } } diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/tnettest.c similarity index 78% rename from tools/shell/src/backup/tnettest.c rename to tools/shell/src/tnettest.c index 772d92d8c6956a3eed1d5273df984c15b73cffc8..9be341225664694d0eb05bb7d76fcfcd1cbe0c30 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/tnettest.c @@ -14,18 +14,20 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #include "taosdef.h" #include "tmsg.h" #include "taoserror.h" #include "tlog.h" #include "tglobal.h" -#include "tsocket.h" #include "trpc.h" #include "rpcHead.h" #include "tchecksum.h" #include "syncMsg.h" +#include "osSocket.h" + #define MAX_PKG_LEN (64 * 1000) #define MAX_SPEED_PKG_LEN (1024 * 1024 * 1024) #define MIN_SPEED_PKG_LEN 1024 @@ -33,7 +35,7 @@ #define MIN_SPEED_PKG_NUM 1 #define BUFFER_SIZE (MAX_PKG_LEN + 1024) -extern int32_t tsRpcMaxUdpSize; +extern int tsRpcMaxUdpSize; typedef struct { char * hostFqdn; @@ -71,15 +73,23 @@ static void *taosNetBindUdpPort(void *sarg) { return NULL; } - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(serverSocket); + return NULL; + } + pSocket->fd = serverSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -98,13 +108,13 @@ static void *taosNetBindUdpPort(void *sarg) { uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); if (iDataNum > 0) { - iDataNum = taosSendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); + iDataNum = taosSendto(pSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); } uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); } - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -132,25 +142,35 @@ static void *taosNetBindTcpPort(void *sarg) { server_addr.sin_addr.s_addr = htonl(INADDR_ANY); int32_t reuse = 1; - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(serverSocket); + return NULL; + } + pSocket->fd = serverSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { uError("failed to bind TCP port:%d since %s", port, strerror(errno)); + taosCloseSocket(&pSocket); return NULL; } - if (taosKeepTcpAlive(serverSocket) < 0) { + if (taosKeepTcpAlive(pSocket) < 0) { uError("failed to set tcp server keep-alive option since %s", strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } if (listen(serverSocket, 10) < 0) { uError("failed to listen TCP port:%d since %s", port, strerror(errno)); + taosCloseSocket(&pSocket); return NULL; } @@ -163,26 +183,26 @@ static void *taosNetBindTcpPort(void *sarg) { continue; } - int32_t ret = taosReadMsg(client, buffer, pinfo->pktLen); + int32_t ret = taosReadMsg(pSocket, buffer, pinfo->pktLen); if (ret < 0 || ret != pinfo->pktLen) { uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); - ret = taosWriteMsg(client, buffer, pinfo->pktLen); + ret = taosWriteMsg(pSocket, buffer, pinfo->pktLen); if (ret < 0) { uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); } - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -196,9 +216,17 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { } int32_t reuse = 1; - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(clientSocket); + return -1; + } + pSocket->fd = clientSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return -1; } @@ -210,27 +238,30 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosKeepTcpAlive(clientSocket); + taosKeepTcpAlive(pSocket); sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port); sprintf(buffer + info->pktLen - 16, "1122334455667788"); - int32_t ret = taosWriteMsg(clientSocket, buffer, info->pktLen); + int32_t ret = taosWriteMsg(pSocket, buffer, info->pktLen); if (ret < 0) { uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - ret = taosReadMsg(clientSocket, buffer, info->pktLen); + ret = taosReadMsg(pSocket, buffer, info->pktLen); if (ret < 0) { uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return 0; } @@ -247,13 +278,23 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { return -1; } - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(clientSocket); + return -1; + } + pSocket->fd = clientSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the send buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); return -1; } - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the receive buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); return -1; } @@ -268,9 +309,10 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); - iDataNum = taosSendto(clientSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); + iDataNum = taosSendto(pSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); if (iDataNum < 0 || iDataNum != info->pktLen) { uError("UDP: failed to perform sendto func since %s", strerror(errno)); + taosCloseSocket(&pSocket); return -1; } @@ -280,10 +322,11 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { if (iDataNum < 0 || iDataNum != info->pktLen) { uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return 0; } @@ -339,7 +382,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) { } static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupReq *pStep) { - SRpcEpSet epSet; + SEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; void * pRpcConn; @@ -352,11 +395,10 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p return TSDB_CODE_RPC_NETWORK_UNAVAIL; } - memset(&epSet, 0, sizeof(SRpcEpSet)); - epSet.inUse = 0; + memset(&epSet, 0, sizeof(SEpSet)); + strcpy(epSet.eps[0].fqdn, serverFqdn); + epSet.eps[0].port = port; epSet.numOfEps = 1; - epSet.port[0] = port; - strcpy(epSet.fqdn[0], serverFqdn); reqMsg.msgType = TDMT_DND_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pktLen); @@ -364,7 +406,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p reqMsg.code = 0; reqMsg.handle = NULL; // rpc handle returned to app reqMsg.ahandle = NULL; // app handle set by client - strcpy(reqMsg.pCont, "nettest"); + strcpy(reqMsg.pCont, "dnode-nettest"); rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); @@ -398,9 +440,9 @@ static int32_t taosNetParseStartup(SStartupReq *pCont) { static void taosNetTestStartup(char *host, int32_t port) { uInfo("check startup, host:%s port:%d\n", host, port); - SStartupReq *pStep = malloc(sizeof(SStartupReq)); + SStartupReq *pStep = taosMemoryMalloc(sizeof(SStartupReq)); while (1) { - int32_t code = taosNetCheckRpc(host, port + TSDB_PORT_DNODEDNODE, 20, 0, pStep); + int32_t code = taosNetCheckRpc(host, port, 20, 0, pStep); if (code > 0) { code = taosNetParseStartup(pStep); } @@ -415,7 +457,7 @@ static void taosNetTestStartup(char *host, int32_t port) { } } - free(pStep); + taosMemoryFree(pStep); } static void taosNetCheckSync(char *host, int32_t port) { @@ -425,8 +467,8 @@ static void taosNetCheckSync(char *host, int32_t port) { return; } - SOCKET connFd = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); - if (connFd < 0) { + TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); + if (pSocket == NULL) { uError("failed to create socket while test port:%d since %s", port, strerror(errno)); return; } @@ -443,63 +485,60 @@ static void taosNetCheckSync(char *host, int32_t port) { pHead->len = sizeof(SSyncMsg) - sizeof(SSyncHead); taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); - if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + if (taosWriteMsg(pSocket, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { uError("failed to test port:%d while send msg since %s", port, strerror(errno)); return; } - if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + if (taosReadMsg(pSocket, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { uError("failed to test port:%d while recv msg since %s", port, strerror(errno)); } uInfo("successed to test TCP port:%d", port); - taosCloseSocket(connFd); + taosCloseSocket(&pSocket); } static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { - int32_t endPort = startPort + TSDB_PORT_SYNC; char spi = 0; - uInfo("check rpc, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); + uInfo("check rpc, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); - for (uint16_t port = startPort; port < endPort; port++) { - int32_t sendpkgLen; - if (pkgLen <= tsRpcMaxUdpSize) { + uint16_t port = startPort; + int32_t sendpkgLen; + if (pkgLen <= tsRpcMaxUdpSize) { sendpkgLen = tsRpcMaxUdpSize + 1000; - } else { + } else { sendpkgLen = pkgLen; - } + } - tsRpcForceTcp = 1; - int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL); - if (ret < 0) { + tsRpcForceTcp = 1; + int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL); + if (ret < 0) { printf("failed to test TCP port:%d\n", port); - } else { + } else { printf("successed to test TCP port:%d\n", port); - } + } - if (pkgLen >= tsRpcMaxUdpSize) { + if (pkgLen >= tsRpcMaxUdpSize) { sendpkgLen = tsRpcMaxUdpSize - 1000; - } else { + } else { sendpkgLen = pkgLen; - } - - tsRpcForceTcp = 0; - ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL); - if (ret < 0) { + } +/* + tsRpcForceTcp = 0; + ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL); + if (ret < 0) { printf("failed to test UDP port:%d\n", port); - } else { + } else { printf("successed to test UDP port:%d\n", port); - } } + */ - taosNetCheckSync(host, startPort + TSDB_PORT_SYNC); - taosNetCheckSync(host, startPort + TSDB_PORT_ARBITRATOR); + taosNetCheckSync(host, startPort); } static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { - int32_t endPort = startPort + 11; - uInfo("work as client, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); + uInfo("work as client, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); uint32_t serverIp = taosGetIpv4FromFqdn(host); if (serverIp == 0xFFFFFFFF) { @@ -508,27 +547,26 @@ static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { } uInfo("server ip:%s is resolved from host:%s", taosIpStr(serverIp), host); - taosNetCheckPort(serverIp, startPort, endPort, pkgLen); + taosNetCheckPort(serverIp, startPort, startPort, pkgLen); } static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { - int32_t endPort = startPort + 11; - uInfo("work as server, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); + uInfo("work as server, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); int32_t port = startPort; - int32_t num = endPort - startPort + 1; + int32_t num = 1; if (num < 0) num = 1; - pthread_t *pids = malloc(2 * num * sizeof(pthread_t)); - STestInfo *tinfos = malloc(num * sizeof(STestInfo)); - STestInfo *uinfos = malloc(num * sizeof(STestInfo)); + TdThread *pids = taosMemoryMalloc(2 * num * sizeof(TdThread)); + STestInfo *tinfos = taosMemoryMalloc(num * sizeof(STestInfo)); + STestInfo *uinfos = taosMemoryMalloc(num * sizeof(STestInfo)); for (int32_t i = 0; i < num; i++) { STestInfo *tcpInfo = tinfos + i; tcpInfo->port = port + i; tcpInfo->pktLen = pkgLen; - if (pthread_create(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { + if (taosThreadCreate(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } @@ -536,15 +574,15 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { STestInfo *udpInfo = uinfos + i; udpInfo->port = port + i; tcpInfo->pktLen = pkgLen; - if (pthread_create(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { + if (taosThreadCreate(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } } for (int32_t i = 0; i < num; i++) { - pthread_join(pids[i], NULL); - pthread_join(pids[(num + i)], NULL); + taosThreadJoin(pids[i], NULL); + taosThreadJoin(pids[(num + i)], NULL); } } @@ -578,7 +616,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, } tsCompressMsgSize = -1; - SRpcEpSet epSet; + SEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; void * pRpcConn; @@ -596,11 +634,10 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, for (int32_t i = 1; i <= pkgNum; i++) { uint64_t startTime = taosGetTimestampUs(); - memset(&epSet, 0, sizeof(SRpcEpSet)); - epSet.inUse = 0; + memset(&epSet, 0, sizeof(SEpSet)); + strcpy(epSet.eps[0].fqdn, host); + epSet.eps[0].port = port; epSet.numOfEps = 1; - epSet.port[0] = port; - strcpy(epSet.fqdn[0], host); reqMsg.msgType = TDMT_DND_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pkgLen); @@ -641,7 +678,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) { - tscEmbedded = 1; + tsLogEmbedded = 1; if (host == NULL) host = tsLocalFqdn; if (port == 0) port = tsServerPort; if (0 == strcmp("speed", role)){ @@ -659,14 +696,14 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, } else if (0 == strcmp("server", role)) { taosNetTestServer(host, port, pkgLen); } else if (0 == strcmp("rpc", role)) { - tscEmbedded = 0; + tsLogEmbedded = 0; taosNetTestRpc(host, port, pkgLen); } else if (0 == strcmp("sync", role)) { taosNetCheckSync(host, port); } else if (0 == strcmp("startup", role)) { taosNetTestStartup(host, port); } else if (0 == strcmp("speed", role)) { - tscEmbedded = 0; + tsLogEmbedded = 0; char type[10] = {0}; taosNetCheckSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType)); }else if (0 == strcmp("fqdn", role)) { @@ -675,5 +712,5 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, taosNetTestStartup(host, port); } - tscEmbedded = 0; + tsLogEmbedded = 0; }