diff --git a/Jenkinsfile2 b/Jenkinsfile2 index ad62fa7044a121819ba27bdb4f47482ca7700fec..441cf896267bed8cd67637c1ea328e884bf0012f 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -35,6 +35,7 @@ def abort_previous(){ def pre_test(){ sh'hostname' sh ''' + date sudo rmtaos || echo "taosd has not installed" ''' sh ''' @@ -60,8 +61,9 @@ def pre_test(){ sh ''' cd ${WKC} git checkout 3.0 + [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. ''' - } + } else{ sh ''' cd ${WKC} @@ -98,7 +100,7 @@ pipeline { } stages { stage('pre_build'){ - agent{label 'slave3_0'} + agent{label " slave3_0 || slave15 || slave16 || slave17 "} options { skipDefaultCheckout() } when { changeRequest() diff --git a/cmake/cmake.define b/cmake/cmake.define index b97c10a4b7ddc2ce1c99283538c76b49bf01e2b3..9c2d5dc04c7471c4d4de54c4f18c6b007e41a40b 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -36,7 +36,14 @@ IF (TD_WINDOWS) ENDIF () ELSE () - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -g3") + +MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") +IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") + ADD_DEFINITIONS("-D_TD_ARM_") +ELSE () + ADD_DEFINITIONS("-msse4.2 -mfma") +ENDIF () ENDIF () diff --git a/cmake/cmake.options b/cmake/cmake.options index 1a1a5b5d785007692fcc5982558024f4bf05b893..946eb5d2583096dab9faecc2e8b222c5c24377f5 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -18,6 +18,13 @@ IF(${TD_WINDOWS}) ON ) + MESSAGE("build iconv Win32") + option( + BUILD_WITH_ICONV + "If build iconv on Windows" + ON + ) + ENDIF () IF(${TD_LINUX} MATCHES TRUE) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 7ef259ba541582baf012e1de4dfbbf3c4f91d5ab..dfb79c0fae948686859a8251ef7e50b1ccbea7b6 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -7,30 +7,52 @@ SET(TD_LINUX FALSE) SET(TD_WINDOWS FALSE) SET(TD_DARWIN FALSE) -IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") +MESSAGE("Compiler ID: ${CMAKE_CXX_COMPILER_ID}") +if(CMAKE_COMPILER_IS_GNUCXX MATCHES 1) + set(CXX_COMPILER_IS_GNU TRUE) +else() + set(CXX_COMPILER_IS_GNU FALSE) +endif() - SET(TD_LINUX TRUE) - SET(OSTYPE "Linux") - ADD_DEFINITIONS("-DLINUX") +MESSAGE("Current system name is ${CMAKE_SYSTEM_NAME}.") - IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) - SET(TD_LINUX_64 TRUE) +IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + + IF (${CXX_COMPILER_IS_GNU}) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99") ELSE () - SET(TD_LINUX_32 TRUE) + ADD_DEFINITIONS("-Wno-tautological-constant-out-of-range-compare -Wno-pointer-sign -Wno-unknown-warning-option") + set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup") + set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") ENDIF () -ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - SET(TD_DARWIN TRUE) - SET(OSTYPE "macOS") - ADD_DEFINITIONS("-DDARWIN") - IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") - MESSAGE("Current system arch is arm64") - SET(TD_DARWIN_64 TRUE) - ADD_DEFINITIONS("-D_TD_DARWIN_64") - ENDIF () + SET(TD_LINUX TRUE) + SET(OSTYPE "Linux") + ADD_DEFINITIONS("-DLINUX") + + IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) + SET(TD_LINUX_64 TRUE) + ELSE () + SET(TD_LINUX_32 TRUE) + ENDIF () + + ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - ADD_DEFINITIONS("-DHAVE_UNISTD_H") + SET(TD_DARWIN TRUE) + SET(OSTYPE "macOS") + ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare -Wno-return-type") + + MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.") + IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") + MESSAGE("Current system arch is arm64") + SET(TD_DARWIN_64 TRUE) + ADD_DEFINITIONS("-D_TD_DARWIN_64") + ENDIF () + + ADD_DEFINITIONS("-DHAVE_UNISTD_H") + ENDIF () ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") @@ -45,6 +67,7 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") SET(TD_WINDOWS_32 TRUE) ADD_DEFINITIONS("-D_TD_WINDOWS_32") ENDIF () + ENDIF() MESSAGE("C Compiler ID: ${CMAKE_C_COMPILER_ID}") diff --git a/cmake/iconv_CMakeLists.txt.in b/cmake/iconv_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..31dfd829fcbc8a5cc6a4b28752eda76280b3c791 --- /dev/null +++ b/cmake/iconv_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# iconv +ExternalProject_Add(iconv + GIT_REPOSITORY https://github.com/win-iconv/win-iconv.git + GIT_TAG v0.0.8 + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/iconv" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 5e2bfc52e1cd9952a76ab2d3709161b9246419aa..9cf68b87f9764b4670cf7091c2f7d83bbb347063 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -83,6 +83,11 @@ if(${BUILD_WITH_NURAFT}) cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_NURAFT}) +# iconv +if(${BUILD_WITH_ICONV}) + cat("${CMAKE_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif(${BUILD_WITH_ICONV}) + # download dependencies configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -127,6 +132,11 @@ target_include_directories( # zlib set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in") add_subdirectory(zlib) +target_include_directories( + zlibstatic + PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib +) target_include_directories( zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib @@ -203,9 +213,10 @@ endif(${BUILD_WITH_TRAFT}) # LIBUV if(${BUILD_WITH_UV}) - if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") - MESSAGE("Windows need set no-sign-compare") - add_compile_options(-Wno-sign-compare) + if (${TD_WINDOWS}) + file(READ "libuv/include/uv.h" CONTENTS) + string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}") + file(WRITE "libuv/include/uv.h" "${CONTENTS_NEW}") endif () add_subdirectory(libuv) endif(${BUILD_WITH_UV}) @@ -238,7 +249,15 @@ if(${BUILD_WITH_SQLITE}) endif(${BUILD_WITH_SQLITE}) # pthread - +if(${BUILD_PTHREAD}) + add_definitions(-DPTW32_STATIC_LIB) + add_subdirectory(pthread) +endif(${BUILD_PTHREAD}) + +# iconv +if(${BUILD_WITH_ICONV}) + add_subdirectory(iconv) +endif(${BUILD_WITH_ICONV}) # ================================================================================================ # Build test diff --git a/include/client/taos.h b/include/client/taos.h index 66e3c491c775791d6a67129f0bd402f5aa1aa898..a102629efa425ca71143669f18091eef073fe414 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -31,27 +31,27 @@ typedef void TAOS_SUB; typedef void **TAOS_ROW; // Data type definition -#define TSDB_DATA_TYPE_NULL 0 // 1 bytes -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_VARCHAR 8 // string, alias for varchar -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // unicode string -#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte -#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes -#define TSDB_DATA_TYPE_UINT 13 // 4 bytes -#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes -#define TSDB_DATA_TYPE_JSON 15 // json string -#define TSDB_DATA_TYPE_VARBINARY 16 // binary -#define TSDB_DATA_TYPE_DECIMAL 17 // decimal -#define TSDB_DATA_TYPE_BLOB 18 // binary +#define TSDB_DATA_TYPE_NULL 0 // 1 bytes +#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes +#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte +#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes +#define TSDB_DATA_TYPE_INT 4 // 4 bytes +#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes +#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes +#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes +#define TSDB_DATA_TYPE_VARCHAR 8 // string, alias for varchar +#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes +#define TSDB_DATA_TYPE_NCHAR 10 // unicode string +#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte +#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes +#define TSDB_DATA_TYPE_UINT 13 // 4 bytes +#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes +#define TSDB_DATA_TYPE_JSON 15 // json string +#define TSDB_DATA_TYPE_VARBINARY 16 // binary +#define TSDB_DATA_TYPE_DECIMAL 17 // decimal +#define TSDB_DATA_TYPE_BLOB 18 // binary #define TSDB_DATA_TYPE_MEDIUMBLOB 19 -#define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string +#define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string typedef enum { TSDB_OPTION_LOCALE, @@ -257,9 +257,15 @@ DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_co void tmqShowMsg(tmq_message_t *tmq_message); int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); -typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); +/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ -DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT* stmt); +DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); +DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message); + +/* ---------------------- OTHER ---------------------------- */ +typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); + +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); #ifdef __cplusplus } diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 23dd3498610810363ebe4cc8ad69b90881936673..385c123fec8f87c7c0785916139d2082e273f33c 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -54,25 +54,25 @@ typedef struct SColumnDataAgg { } SColumnDataAgg; typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t rowSize; - int32_t numOfCols; + STimeWindow window; + int32_t rows; + int32_t rowSize; + int16_t numOfCols; + int16_t hasVarCol; union {int64_t uid; int64_t blockId;}; } SDataBlockInfo; -typedef struct SConstantItem { - SColumnInfo info; - int32_t startRow; // run-length-encoding to save the space for multiple rows - int32_t endRow; - SVariant value; -} SConstantItem; +//typedef struct SConstantItem { +// SColumnInfo info; +// int32_t startRow; // run-length-encoding to save the space for multiple rows +// int32_t endRow; +// SVariant value; +//} SConstantItem; // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; SArray *pDataBlock; // SArray - SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. SDataBlockInfo info; } SSDataBlock; @@ -94,63 +94,15 @@ typedef struct SColumnInfoData { }; } SColumnInfoData; -static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { - int64_t tbUid = pBlock->info.uid; - int32_t numOfCols = pBlock->info.numOfCols; - int32_t rows = pBlock->info.rows; - int32_t sz = taosArrayGetSize(pBlock->pDataBlock); - - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, tbUid); - tlen += taosEncodeFixedI32(buf, numOfCols); - tlen += taosEncodeFixedI32(buf, rows); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); - tlen += taosEncodeFixedI16(buf, pColData->info.colId); - tlen += taosEncodeFixedI16(buf, pColData->info.type); - tlen += taosEncodeFixedI32(buf, pColData->info.bytes); - int32_t colSz = rows * pColData->info.bytes; - tlen += taosEncodeBinary(buf, pColData->pData, colSz); - } - return tlen; -} - -static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { - int32_t sz; - - buf = taosDecodeFixedI64(buf, &pBlock->info.uid); - buf = taosDecodeFixedI32(buf, &pBlock->info.numOfCols); - buf = taosDecodeFixedI32(buf, &pBlock->info.rows); - buf = taosDecodeFixedI32(buf, &sz); - pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < sz; i++) { - SColumnInfoData data = {0}; - buf = taosDecodeFixedI16(buf, &data.info.colId); - buf = taosDecodeFixedI16(buf, &data.info.type); - buf = taosDecodeFixedI32(buf, &data.info.bytes); - int32_t colSz = pBlock->info.rows * data.info.bytes; - buf = taosDecodeBinary(buf, (void**)&data.pData, colSz); - taosArrayPush(pBlock->pDataBlock, &data); - } - return (void*)buf; -} +void* blockDataDestroy(SSDataBlock* pBlock); +int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); +void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { if (pBlock == NULL) { return; } - - // int32_t numOfOutput = pBlock->info.numOfCols; - int32_t sz = taosArrayGetSize(pBlock->pDataBlock); - for (int32_t i = 0; i < sz; ++i) { - SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); - tfree(pColInfoData->pData); - } - - taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockAgg); - // tfree(pBlock); + blockDataDestroy(pBlock); } static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index a9a056aab80cd56f3a744d44a53daa11d6cecf74..67946c9218b861a6baa4a128a44803a71aad128d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -52,6 +52,21 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) +static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { + if (!pColumnInfoData->hasNull) { + return false; + } + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + return pColumnInfoData->varmeta.offset[row] == -1; + } else { + if (pColumnInfoData->nullbitmap == NULL) { + return false; + } + + return colDataIsNull_f(pColumnInfoData->nullbitmap, row); + } +} + static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { if (!pColumnInfoData->hasNull) { @@ -79,10 +94,10 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } } -#define BitmapLen(_n) (((_n) + ((1<> NBIT) - +#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) -#define colDataGetData(p1_, r_) \ +// SColumnInfoData, rowNumber +#define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) @@ -116,7 +131,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol); +void blockDataClearup(SSDataBlock* pDataBlock); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); void* blockDataDestroy(SSDataBlock* pBlock); @@ -125,4 +140,4 @@ void* blockDataDestroy(SSDataBlock* pBlock); } #endif -#endif /*_TD_COMMON_EP_H_*/ +#endif /*_TD_COMMON_EP_H_*/ diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 342d78382ab4ac680eca259739c3061b15b94a22..90aac6edcdec34c4efee9abacb995c7d3827f9f3 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -51,6 +51,7 @@ extern int32_t tsCompatibleModel; extern bool tsEnableSlaveQuery; extern bool tsPrintAuth; extern int64_t tsTickPerDay[3]; +extern int32_t tsMultiProcess; // monitor extern bool tsEnableMonitor; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0e0018b850c6323634f30d40f69e36b26f903f9c..92698586450127aa00317b5892c26d1ff0641460 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -25,6 +25,7 @@ #include "tlist.h" #include "trow.h" #include "tname.h" +#include "tuuid.h" #ifdef __cplusplus extern "C" { @@ -111,15 +112,16 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_MAX, } EShowType; -#define TSDB_ALTER_TABLE_ADD_TAG 1 -#define TSDB_ALTER_TABLE_DROP_TAG 2 -#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 -#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 - +#define TSDB_ALTER_TABLE_ADD_TAG 1 +#define TSDB_ALTER_TABLE_DROP_TAG 2 +#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 +#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 #define TSDB_ALTER_TABLE_ADD_COLUMN 5 #define TSDB_ALTER_TABLE_DROP_COLUMN 6 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7 #define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 +#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9 +#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10 #define TSDB_FILL_NONE 0 #define TSDB_FILL_NULL 1 @@ -172,7 +174,7 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t dbId; int32_t vgVersion; - int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SBuildUseDBInput; typedef struct SField { @@ -191,9 +193,17 @@ typedef struct SEp { typedef struct { int32_t contLen; - int32_t vgId; + union { + int32_t vgId; + int32_t streamTaskId; + }; } SMsgHead; +typedef struct { + int32_t workerType; + int32_t streamTaskId; +} SStreamExecMsgHead; + // Submit message for one table typedef struct SSubmitBlk { int64_t uid; // table unique id @@ -416,7 +426,7 @@ typedef struct { }; } SColumnFilterList; /* - * for client side struct, we only need the column id, type, bytes are not necessary + * for client side struct, only column id, type, bytes are necessary * But for data in vnode side, we need all the following information. */ typedef struct { @@ -425,10 +435,10 @@ typedef struct { int16_t slotId; }; - int16_t type; - int32_t bytes; - uint8_t precision; - uint8_t scale; + int16_t type; + int32_t bytes; + uint8_t precision; + uint8_t scale; } SColumnInfo; typedef struct { @@ -529,7 +539,7 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t dbId; int32_t vgVersion; - int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SUseDbReq; int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); @@ -556,15 +566,13 @@ int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); typedef struct { - SArray *epSetList; // SArray + SArray* epSetList; // SArray } SQnodeListRsp; int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); void tFreeSQnodeListRsp(SQnodeListRsp* pRsp); - - typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; @@ -695,6 +703,7 @@ typedef struct { int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); int32_t tDeserializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); +void tFreeSStatusRsp(SStatusRsp* pRsp); typedef struct { int32_t reserved; @@ -780,7 +789,6 @@ typedef struct SVgroupInfo { int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SVgroupInfo; - typedef struct { int32_t numOfVgroups; SVgroupInfo vgroups[]; @@ -1077,8 +1085,8 @@ typedef struct { } STaskStatus; typedef struct { - int64_t refId; - SArray *taskStatus; //SArray + int64_t refId; + SArray* taskStatus; // SArray } SSchedulerStatusRsp; typedef struct { @@ -1087,35 +1095,31 @@ typedef struct { int8_t action; } STaskAction; - typedef struct SQueryNodeEpId { int32_t nodeId; // vgId or qnodeId SEp ep; } SQueryNodeEpId; - typedef struct { SMsgHead header; uint64_t sId; SQueryNodeEpId epId; - SArray *taskAction; //SArray + SArray* taskAction; // SArray } SSchedulerHbReq; -int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq); -int32_t tDeserializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq); -void tFreeSSchedulerHbReq(SSchedulerHbReq *pReq); - +int32_t tSerializeSSchedulerHbReq(void* buf, int32_t bufLen, SSchedulerHbReq* pReq); +int32_t tDeserializeSSchedulerHbReq(void* buf, int32_t bufLen, SSchedulerHbReq* pReq); +void tFreeSSchedulerHbReq(SSchedulerHbReq* pReq); typedef struct { uint64_t seqId; SQueryNodeEpId epId; - SArray *taskStatus; //SArray + SArray* taskStatus; // SArray } SSchedulerHbRsp; -int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp); -int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp); -void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp); - +int32_t tSerializeSSchedulerHbRsp(void* buf, int32_t bufLen, SSchedulerHbRsp* pRsp); +int32_t tDeserializeSSchedulerHbRsp(void* buf, int32_t bufLen, SSchedulerHbRsp* pRsp); +void tFreeSSchedulerHbRsp(SSchedulerHbRsp* pRsp); typedef struct { SMsgHead header; @@ -1172,8 +1176,8 @@ typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; int8_t igExists; char* sql; - char* physicalPlan; - char* logicalPlan; + char* ast; + char subscribeDbName[TSDB_DB_NAME_LEN]; } SCMCreateTopicReq; int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); @@ -1387,7 +1391,6 @@ typedef struct SVCreateTbReq { typedef struct { int32_t code; - int tmp; // TODO: to avoid compile error } SVCreateTbRsp, SVUpdateTbRsp; int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); @@ -1403,7 +1406,6 @@ void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); typedef struct { SArray* rspList; // SArray - int tmp; // TODO: to avoid compile error } SVCreateTbBatchRsp; int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp); @@ -1418,7 +1420,7 @@ typedef struct { } SVDropTbReq; typedef struct { - int tmp; // TODO: to avoid compile error + int tmp; // TODO: to avoid compile error } SVDropTbRsp; int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq); @@ -1918,32 +1920,18 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) } return buf; } -typedef enum { - TD_TIME_UNIT_UNKNOWN = -1, - TD_TIME_UNIT_YEAR = 0, - TD_TIME_UNIT_SEASON = 1, - TD_TIME_UNIT_MONTH = 2, - TD_TIME_UNIT_WEEK = 3, - TD_TIME_UNIT_DAY = 4, - TD_TIME_UNIT_HOUR = 5, - TD_TIME_UNIT_MINUTE = 6, - TD_TIME_UNIT_SEC = 7, - TD_TIME_UNIT_MILLISEC = 8, - TD_TIME_UNIT_MICROSEC = 9, - TD_TIME_UNIT_NANOSEC = 10 -} ETDTimeUnit; - -typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; - int8_t slidingUnit; +typedef struct { + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX char indexName[TSDB_INDEX_NAME_LEN]; - char timezone[TD_TIMEZONE_LEN]; // sma data is invalid if timezone change. - uint16_t exprLen; - uint16_t tagsFilterLen; + char timezone[TD_TIMEZONE_LEN]; // sma data expired if timezone changes. + int32_t exprLen; + int32_t tagsFilterLen; int64_t indexUid; tb_uid_t tableUid; // super/child/common table uid int64_t interval; + int64_t offset; // use unit by precision of DB int64_t sliding; char* expr; // sma expression char* tagsFilter; @@ -1955,7 +1943,7 @@ typedef struct { } SVCreateTSmaReq; typedef struct { - int8_t type; // 0 status report, 1 update data + int8_t type; // 0 status report, 1 update data char indexName[TSDB_INDEX_NAME_LEN]; // STimeWindow windows; } STSmaMsg; @@ -1966,7 +1954,7 @@ typedef struct { } SVDropTSmaReq; typedef struct { - int tmp; // TODO: to avoid compile error + int tmp; // TODO: to avoid compile error } SVCreateTSmaRsp, SVDropTSmaRsp; int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq); @@ -1988,7 +1976,7 @@ typedef struct { typedef struct { int64_t indexUid; - TSKEY skey; // startTS of one interval/sliding + TSKEY skey; // startKey of one interval/sliding window int64_t interval; int32_t dataLen; // not including head int8_t intervalUnit; @@ -2045,13 +2033,14 @@ static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { tlen += taosEncodeFixedI8(buf, pSma->slidingUnit); tlen += taosEncodeString(buf, pSma->indexName); tlen += taosEncodeString(buf, pSma->timezone); - tlen += taosEncodeFixedU16(buf, pSma->exprLen); - tlen += taosEncodeFixedU16(buf, pSma->tagsFilterLen); + tlen += taosEncodeFixedI32(buf, pSma->exprLen); + tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen); tlen += taosEncodeFixedI64(buf, pSma->indexUid); tlen += taosEncodeFixedI64(buf, pSma->tableUid); tlen += taosEncodeFixedI64(buf, pSma->interval); + tlen += taosEncodeFixedI64(buf, pSma->offset); tlen += taosEncodeFixedI64(buf, pSma->sliding); - + if (pSma->exprLen > 0) { tlen += taosEncodeString(buf, pSma->expr); } @@ -2079,36 +2068,28 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { buf = taosDecodeFixedI8(buf, &pSma->slidingUnit); buf = taosDecodeStringTo(buf, pSma->indexName); buf = taosDecodeStringTo(buf, pSma->timezone); - buf = taosDecodeFixedU16(buf, &pSma->exprLen); - buf = taosDecodeFixedU16(buf, &pSma->tagsFilterLen); + buf = taosDecodeFixedI32(buf, &pSma->exprLen); + buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen); buf = taosDecodeFixedI64(buf, &pSma->indexUid); buf = taosDecodeFixedI64(buf, &pSma->tableUid); buf = taosDecodeFixedI64(buf, &pSma->interval); + buf = taosDecodeFixedI64(buf, &pSma->offset); buf = taosDecodeFixedI64(buf, &pSma->sliding); - if (pSma->exprLen > 0) { - pSma->expr = (char*)calloc(pSma->exprLen, 1); - if (pSma->expr != NULL) { - buf = taosDecodeStringTo(buf, pSma->expr); - } else { + if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) { tdDestroyTSma(pSma); return NULL; } - } else { pSma->expr = NULL; } if (pSma->tagsFilterLen > 0) { - pSma->tagsFilter = (char*)calloc(pSma->tagsFilterLen, 1); - if (pSma->tagsFilter != NULL) { - buf = taosDecodeStringTo(buf, pSma->tagsFilter); - } else { + if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) { tdDestroyTSma(pSma); return NULL; } - } else { pSma->tagsFilter = NULL; } @@ -2199,13 +2180,36 @@ typedef struct { SArray* topics; // SArray } SMqCMGetSubEpRsp; -struct tmq_message_t { +typedef struct { SMqRspHead head; union { SMqPollRsp consumeRsp; SMqCMGetSubEpRsp getEpRsp; }; void* extra; +} SMqMsgWrapper; + +typedef struct { + int32_t curBlock; + int32_t curRow; + void** uData; +} SMqRowIter; + +struct tmq_message_t_v1 { + SMqPollRsp rsp; + SMqRowIter iter; +}; + +struct tmq_message_t { + SMqRspHead head; + union { + SMqPollRsp consumeRsp; + SMqCMGetSubEpRsp getEpRsp; + }; + void* extra; + int32_t curBlock; + int32_t curRow; + void** uData; }; static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } @@ -2287,8 +2291,65 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p return buf; } +enum { + STREAM_TASK_STATUS__RUNNING = 1, + STREAM_TASK_STATUS__STOP, +}; + +typedef struct { + int64_t streamId; + int32_t taskId; + int32_t level; + int8_t status; + char* qmsg; + void* executor; + // void* stateStore; + // storage handle +} SStreamTask; + +static FORCE_INLINE SStreamTask* streamTaskNew(int64_t streamId, int32_t level) { + SStreamTask* pTask = (SStreamTask*)calloc(1, sizeof(SStreamTask)); + if (pTask == NULL) { + return NULL; + } + pTask->taskId = tGenIdPI32(); + pTask->status = STREAM_TASK_STATUS__RUNNING; + pTask->qmsg = NULL; + return pTask; +} + +int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask); +int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask); +void tFreeSStreamTask(SStreamTask* pTask); + +typedef struct { + SMsgHead head; + SStreamTask* task; +} SStreamTaskDeployReq; + +typedef struct { + int32_t reserved; +} SStreamTaskDeployRsp; + +typedef struct { + SStreamExecMsgHead head; + // TODO: other info needed by task +} SStreamTaskExecReq; + +typedef struct { + int32_t reserved; +} SStreamTaskExecRsp; + #pragma pack(pop) +struct SRpcMsg; +struct SEpSet; +struct SMgmtWrapper; +typedef int32_t (*PutToQueueFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); +typedef int32_t (*SendReqFp)(struct SMgmtWrapper* pWrapper, struct SEpSet* epSet, struct SRpcMsg* rpcMsg); +typedef int32_t (*SendMnodeReqFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* rpcMsg); +typedef void (*SendRspFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* rpcMsg); + #ifdef __cplusplus } #endif diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 03f8daad42e1106b2acecfc2097c456fc049c7af..a596794b3d9ac493f775bd2d6f6bb74f12075036 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -189,6 +189,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) @@ -199,6 +200,8 @@ enum { // Requests handled by SNODE TD_NEW_MSG_SEG(TDMT_SND_MSG) + TD_DEF_MSG_TYPE(TDMT_SND_TASK_DEPLOY, "snode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) + TD_DEF_MSG_TYPE(TDMT_SND_TASK_EXEC, "snode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) #if defined(TD_MSG_NUMBER_) TDMT_MAX diff --git a/include/common/trow.h b/include/common/trow.h index 4dd3daba4dc13aa15a764206ab8fdb19b45e4dcc..47edf6f1ad45b2181ef75f25d78929ceafb58be8 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -98,13 +98,13 @@ typedef void *SRow; typedef struct { TDRowValT valType; - void *val; + void * val; } SCellVal; typedef struct { // TODO - int tmp; // TODO: to avoid compile error -} STpRow; // tuple + int tmp; // TODO: to avoid compile error +} STpRow; // tuple #pragma pack(push, 1) typedef struct { @@ -158,8 +158,8 @@ typedef struct { int16_t nBitmaps; int16_t nBoundBitmaps; int32_t offset; - void *pBitmap; - void *pOffset; + void * pBitmap; + void * pOffset; int32_t extendedRowSize; } SRowBuilder; @@ -273,7 +273,7 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); @@ -311,7 +311,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pValType = (((*pDestByte) & 0xC0) >> 6); @@ -617,7 +617,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa if (tdValIsNorm(valType, val, colType)) { // ts key stored in STSRow.ts SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN @@ -635,7 +635,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa // NULL/None value else { SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN const void *nullVal = getNullValue(colType); @@ -697,9 +697,9 @@ static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, int16_t c } // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. if (TD_IS_TP_ROW(pRow)) { - tdAppendColValToTpRow(pBuilder, valType, val, true, colType, colIdx, offset); + tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); } else { - tdAppendColValToKvRow(pBuilder, valType, val, true, colType, colIdx, offset, colId); + tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId); } return TSDB_CODE_SUCCESS; } @@ -771,8 +771,8 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v typedef struct { STSchema *pSchema; - STSRow *pRow; - void *pBitmap; + STSRow * pRow; + void * pBitmap; uint32_t offset; col_id_t maxColId; col_id_t colIdx; // [PRIMARYKEY_TIMESTAMP_COL_ID, nSchemaCols], PRIMARYKEY_TIMESTAMP_COL_ID equals 1 @@ -877,7 +877,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy // internal static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; + STSRow * pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; col_id_t kvNCols = tdRowGetNCols(pRow); @@ -1068,7 +1068,7 @@ typedef struct { typedef struct { STSchema *pSchema; - STSRow *pRow; + STSRow * pRow; } STSRowReader; typedef struct { diff --git a/include/common/ttime.h b/include/common/ttime.h index b71426e31261c15486b320dac9040ecc84324414..57af24e6353ad221d9a33bd83fe91f0a89af7d2b 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -25,6 +25,17 @@ extern "C" { #define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') +#define TIME_UNIT_NANOSECOND 'b' +#define TIME_UNIT_MICROSECOND 'u' +#define TIME_UNIT_MILLISECOND 'a' +#define TIME_UNIT_SECOND 's' +#define TIME_UNIT_MINUTE 'm' +#define TIME_UNIT_HOUR 'h' +#define TIME_UNIT_DAY 'd' +#define TIME_UNIT_WEEK 'w' +#define TIME_UNIT_MONTH 'n' +#define TIME_UNIT_YEAR 'y' + /* * @return timestamp decided by global conf variable, tsTimePrecision * if precision == TSDB_TIME_PRECISION_MICRO, it returns timestamp in microsecond. diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 4e3066790af0faf416b5687ff24b7c04cffd3333..da34ca62536872599cb0e91db172d30a83e73346 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -34,127 +34,150 @@ #define TK_NK_REM 16 #define TK_NK_CONCAT 17 #define TK_CREATE 18 -#define TK_USER 19 -#define TK_PASS 20 -#define TK_NK_STRING 21 -#define TK_ALTER 22 -#define TK_PRIVILEGE 23 -#define TK_DROP 24 -#define TK_DNODE 25 -#define TK_PORT 26 -#define TK_NK_INTEGER 27 -#define TK_NK_ID 28 -#define TK_NK_IPTOKEN 29 -#define TK_DATABASE 30 -#define TK_USE 31 -#define TK_IF 32 -#define TK_NOT 33 -#define TK_EXISTS 34 -#define TK_BLOCKS 35 -#define TK_CACHE 36 -#define TK_CACHELAST 37 -#define TK_COMP 38 -#define TK_DAYS 39 -#define TK_FSYNC 40 -#define TK_MAXROWS 41 -#define TK_MINROWS 42 -#define TK_KEEP 43 -#define TK_PRECISION 44 -#define TK_QUORUM 45 -#define TK_REPLICA 46 -#define TK_TTL 47 -#define TK_WAL 48 -#define TK_VGROUPS 49 -#define TK_SINGLE_STABLE 50 -#define TK_STREAM_MODE 51 -#define TK_TABLE 52 -#define TK_NK_LP 53 -#define TK_NK_RP 54 -#define TK_STABLE 55 -#define TK_USING 56 -#define TK_TAGS 57 -#define TK_NK_DOT 58 -#define TK_NK_COMMA 59 -#define TK_COMMENT 60 -#define TK_BOOL 61 -#define TK_TINYINT 62 -#define TK_SMALLINT 63 -#define TK_INT 64 -#define TK_INTEGER 65 -#define TK_BIGINT 66 -#define TK_FLOAT 67 -#define TK_DOUBLE 68 -#define TK_BINARY 69 -#define TK_TIMESTAMP 70 -#define TK_NCHAR 71 -#define TK_UNSIGNED 72 -#define TK_JSON 73 -#define TK_VARCHAR 74 -#define TK_MEDIUMBLOB 75 -#define TK_BLOB 76 -#define TK_VARBINARY 77 -#define TK_DECIMAL 78 -#define TK_SMA 79 -#define TK_SHOW 80 -#define TK_DNODES 81 -#define TK_USERS 82 -#define TK_DATABASES 83 -#define TK_TABLES 84 -#define TK_STABLES 85 -#define TK_MNODES 86 -#define TK_MODULES 87 -#define TK_QNODES 88 -#define TK_FUNCTIONS 89 -#define TK_INDEXES 90 -#define TK_FROM 91 -#define TK_STREAMS 92 -#define TK_LIKE 93 -#define TK_NK_FLOAT 94 -#define TK_NK_BOOL 95 -#define TK_NK_VARIABLE 96 -#define TK_BETWEEN 97 -#define TK_IS 98 -#define TK_NULL 99 -#define TK_NK_LT 100 -#define TK_NK_GT 101 -#define TK_NK_LE 102 -#define TK_NK_GE 103 -#define TK_NK_NE 104 -#define TK_NK_EQ 105 -#define TK_MATCH 106 -#define TK_NMATCH 107 -#define TK_IN 108 -#define TK_AS 109 -#define TK_JOIN 110 -#define TK_ON 111 -#define TK_INNER 112 -#define TK_SELECT 113 -#define TK_DISTINCT 114 -#define TK_WHERE 115 -#define TK_PARTITION 116 -#define TK_BY 117 -#define TK_SESSION 118 -#define TK_STATE_WINDOW 119 -#define TK_INTERVAL 120 -#define TK_SLIDING 121 -#define TK_FILL 122 -#define TK_VALUE 123 -#define TK_NONE 124 -#define TK_PREV 125 -#define TK_LINEAR 126 -#define TK_NEXT 127 -#define TK_GROUP 128 -#define TK_HAVING 129 -#define TK_ORDER 130 -#define TK_SLIMIT 131 -#define TK_SOFFSET 132 -#define TK_LIMIT 133 -#define TK_OFFSET 134 -#define TK_ASC 135 -#define TK_DESC 136 -#define TK_NULLS 137 -#define TK_FIRST 138 -#define TK_LAST 139 +#define TK_ACCOUNT 19 +#define TK_NK_ID 20 +#define TK_PASS 21 +#define TK_NK_STRING 22 +#define TK_ALTER 23 +#define TK_PPS 24 +#define TK_TSERIES 25 +#define TK_STORAGE 26 +#define TK_STREAMS 27 +#define TK_QTIME 28 +#define TK_DBS 29 +#define TK_USERS 30 +#define TK_CONNS 31 +#define TK_STATE 32 +#define TK_USER 33 +#define TK_PRIVILEGE 34 +#define TK_DROP 35 +#define TK_DNODE 36 +#define TK_PORT 37 +#define TK_NK_INTEGER 38 +#define TK_DNODES 39 +#define TK_NK_IPTOKEN 40 +#define TK_LOCAL 41 +#define TK_QNODE 42 +#define TK_ON 43 +#define TK_DATABASE 44 +#define TK_USE 45 +#define TK_IF 46 +#define TK_NOT 47 +#define TK_EXISTS 48 +#define TK_BLOCKS 49 +#define TK_CACHE 50 +#define TK_CACHELAST 51 +#define TK_COMP 52 +#define TK_DAYS 53 +#define TK_FSYNC 54 +#define TK_MAXROWS 55 +#define TK_MINROWS 56 +#define TK_KEEP 57 +#define TK_PRECISION 58 +#define TK_QUORUM 59 +#define TK_REPLICA 60 +#define TK_TTL 61 +#define TK_WAL 62 +#define TK_VGROUPS 63 +#define TK_SINGLE_STABLE 64 +#define TK_STREAM_MODE 65 +#define TK_RETENTIONS 66 +#define TK_FILE_FACTOR 67 +#define TK_NK_FLOAT 68 +#define TK_TABLE 69 +#define TK_NK_LP 70 +#define TK_NK_RP 71 +#define TK_STABLE 72 +#define TK_ADD 73 +#define TK_COLUMN 74 +#define TK_MODIFY 75 +#define TK_RENAME 76 +#define TK_TAG 77 +#define TK_SET 78 +#define TK_NK_EQ 79 +#define TK_USING 80 +#define TK_TAGS 81 +#define TK_NK_DOT 82 +#define TK_NK_COMMA 83 +#define TK_COMMENT 84 +#define TK_BOOL 85 +#define TK_TINYINT 86 +#define TK_SMALLINT 87 +#define TK_INT 88 +#define TK_INTEGER 89 +#define TK_BIGINT 90 +#define TK_FLOAT 91 +#define TK_DOUBLE 92 +#define TK_BINARY 93 +#define TK_TIMESTAMP 94 +#define TK_NCHAR 95 +#define TK_UNSIGNED 96 +#define TK_JSON 97 +#define TK_VARCHAR 98 +#define TK_MEDIUMBLOB 99 +#define TK_BLOB 100 +#define TK_VARBINARY 101 +#define TK_DECIMAL 102 +#define TK_SMA 103 +#define TK_ROLLUP 104 +#define TK_SHOW 105 +#define TK_DATABASES 106 +#define TK_TABLES 107 +#define TK_STABLES 108 +#define TK_MNODES 109 +#define TK_MODULES 110 +#define TK_QNODES 111 +#define TK_FUNCTIONS 112 +#define TK_INDEXES 113 +#define TK_FROM 114 +#define TK_LIKE 115 +#define TK_INDEX 116 +#define TK_FULLTEXT 117 +#define TK_FUNCTION 118 +#define TK_INTERVAL 119 +#define TK_TOPIC 120 +#define TK_AS 121 +#define TK_NK_BOOL 122 +#define TK_NK_VARIABLE 123 +#define TK_BETWEEN 124 +#define TK_IS 125 +#define TK_NULL 126 +#define TK_NK_LT 127 +#define TK_NK_GT 128 +#define TK_NK_LE 129 +#define TK_NK_GE 130 +#define TK_NK_NE 131 +#define TK_MATCH 132 +#define TK_NMATCH 133 +#define TK_IN 134 +#define TK_JOIN 135 +#define TK_INNER 136 +#define TK_SELECT 137 +#define TK_DISTINCT 138 +#define TK_WHERE 139 +#define TK_PARTITION 140 +#define TK_BY 141 +#define TK_SESSION 142 +#define TK_STATE_WINDOW 143 +#define TK_SLIDING 144 +#define TK_FILL 145 +#define TK_VALUE 146 +#define TK_NONE 147 +#define TK_PREV 148 +#define TK_LINEAR 149 +#define TK_NEXT 150 +#define TK_GROUP 151 +#define TK_HAVING 152 +#define TK_ORDER 153 +#define TK_SLIMIT 154 +#define TK_SOFFSET 155 +#define TK_LIMIT 156 +#define TK_OFFSET 157 +#define TK_ASC 158 +#define TK_DESC 159 +#define TK_NULLS 160 +#define TK_FIRST 161 +#define TK_LAST 162 #define TK_NK_SPACE 300 #define TK_NK_COMMENT 301 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 5aa22bdcad699e7502d880786372d31a3545bd56..a936ea3b5410a4942bd8775a169e7841bc0083cd 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -27,7 +27,7 @@ extern "C" { typedef int32_t VarDataOffsetT; typedef uint32_t TDRowLenT; typedef uint8_t TDRowValT; -typedef uint16_t col_id_t; +typedef int16_t col_id_t; typedef int8_t col_type_t; #pragma pack(push, 1) diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 995015fe6301b950f2ae415aef791296e6621d28..63f305ab2da00915555dbe63ffbdaefb4d2af73c 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -31,7 +31,7 @@ typedef struct SVariant { uint64_t u; double d; char *pz; - wchar_t *wpz; + TdUcs4 *ucs4; SArray *arr; // only for 'in' query to hold value list, not value for a field }; } SVariant; diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 7f7b8a8d53f1fa249704d6e5510273ae4247da4a..84c2ea55f7b361b14f957d377a38dc4afdbd6605 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -21,24 +21,19 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SBnode SBnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SBnode SBnode; typedef struct { - int64_t numOfErrors; } SBnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + SMgmtWrapper *pWrapper; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SBnodeOpt; /* ------------------------ SBnode ------------------------ */ @@ -76,13 +71,6 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad); */ int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs); -/** - * @brief Drop a bnode. - * - * @param path Path of the bnode. - */ -void bndDestroy(const char *path); - #ifdef __cplusplus } #endif diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 068e9a69170bfd1659b2961bef220b765d74c35e..8d19ce23dfdfce22a3e0e9b0961f58aa9319bc74 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -33,8 +33,7 @@ typedef struct SDnode SDnode; int32_t dndInit(); /** - * @brief clear the environment - * + * @brief Clear the environment */ void dndCleanup(); @@ -42,22 +41,24 @@ void dndCleanup(); typedef struct { int32_t numOfSupportVnodes; uint16_t serverPort; - char dataDir[TSDB_FILENAME_LEN]; + char dataDir[PATH_MAX]; char localEp[TSDB_EP_LEN]; char localFqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; char secondEp[TSDB_EP_LEN]; SDiskCfg *pDisks; int32_t numOfDisks; -} SDnodeObjCfg; +} SDnodeOpt; + +typedef enum { DND_EVENT_START, DND_EVENT_STOP = 1, DND_EVENT_RELOAD } EDndEvent; /** * @brief Initialize and start the dnode. * - * @param pCfg Config of the dnode. + * @param pOption Option of the dnode. * @return SDnode* The dnode object. */ -SDnode *dndCreate(SDnodeObjCfg *pCfg); +SDnode *dndCreate(const SDnodeOpt *pOption); /** * @brief Stop and cleanup the dnode. @@ -66,6 +67,21 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg); */ void dndClose(SDnode *pDnode); +/** + * @brief Run dnode until specific event is receive. + * + * @param pDnode The dnode object to run. + */ +int32_t dndRun(SDnode *pDnode); + +/** + * @brief Handle event in the dnode. + * + * @param pDnode The dnode object to close. + * @param event The event to handle. + */ +void dndHandleEvent(SDnode *pDnode, EDndEvent event); + #ifdef __cplusplus } #endif diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index d295e772e898e62bce7e9b822309a79640368a0d..d421b2e45d1ea0751aa3aaff0b42144ff293e247 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -23,27 +23,21 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SMnode SMnode; -typedef struct SMnodeMsg SMnodeMsg; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SMnode SMnode; typedef struct { - int32_t dnodeId; - int64_t clusterId; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; - SDnode *pDnode; - PutReqToMWriteQFp putReqToMWriteQFp; - PutReqToMReadQFp putReqToMReadQFp; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; + SMgmtWrapper *pWrapper; + PutToQueueFp putToWriteQFp; + PutToQueueFp putToReadQFp; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SMnodeOpt; /* ------------------------ SMnode ------------------------ */ @@ -73,11 +67,11 @@ void mndClose(SMnode *pMnode); int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption); /** - * @brief Drop a mnode. + * @brief Start mnode * - * @param path Path of the mnode. + * @param pMnode The mnode object. */ -void mndDestroy(const char *path); +int32_t mndStart(SMnode *pMnode); /** * @brief Get mnode monitor info. @@ -104,37 +98,13 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr */ int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); -/** - * @brief Initialize mnode msg. - * - * @param pMnode The mnode object. - * @param pMsg The request rpc msg. - * @return int32_t The created mnode msg. - */ -SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg); - -/** - * @brief Cleanup mnode msg. - * - * @param pMsg The request msg. - */ -void mndCleanupMsg(SMnodeMsg *pMsg); - -/** - * @brief Cleanup mnode msg. - * - * @param pMsg The request msg. - * @param code The error code. - */ -void mndSendRsp(SMnodeMsg *pMsg, int32_t code); - /** * @brief Process the read, write, sync request. * * @param pMsg The request msg. * @return int32_t 0 for success, -1 for failure. */ -void mndProcessMsg(SMnodeMsg *pMsg); +int32_t mndProcessMsg(SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 3de2986047c4b5bf1e047e294b46999149abd433..df735cf1b3c97f5e64ad57d37bdc8a587fc529b1 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -21,11 +21,8 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SQnode SQnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SQnode SQnode; typedef struct { int64_t numOfStartTask; @@ -39,13 +36,12 @@ typedef struct { } SQnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + SMgmtWrapper *pWrapper; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SQnodeOpt; /* ------------------------ SQnode ------------------------ */ diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index 21a93532e0d77b3c8f5e6157380e7804b393d622..4202859359e2fcb014b662956dcdc973f54a8c6c 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -16,6 +16,7 @@ #ifndef _TD_SNODE_H_ #define _TD_SNODE_H_ +#include "tcommon.h" #include "tmsg.h" #include "trpc.h" @@ -24,24 +25,19 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SDnode SDnode; -typedef struct SSnode SSnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SSnode SSnode; typedef struct { - int64_t numOfErrors; } SSnodeLoad; typedef struct { - int32_t sver; - int32_t dnodeId; - int64_t clusterId; - SDnode *pDnode; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + int32_t dnodeId; + int64_t clusterId; + SMgmtWrapper *pWrapper; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SSnodeOpt; /* ------------------------ SSnode ------------------------ */ @@ -76,20 +72,9 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad); * @param pSnode The snode object. * @param pMsg The request message * @param pRsp The response message - * @return int32_t 0 for success, -1 for failure */ -int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); - -int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); - -int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg); - -/** - * @brief Drop a snode. - * - * @param path Path of the snode. - */ -void sndDestroy(const char *path); +void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg); +void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/include/libs/function/function.h b/include/libs/function/function.h index fde09e59daf214791d2b72fc291a435d2635c40b..e33805437a81b2e96b9212c5563ef3f9944cb8cb 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -295,19 +295,8 @@ typedef struct SMultiFunctionsDesc { int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, bool isSuperTable); -/** - * If the given name is a valid built-in sql function, the value of true will be returned. - * @param name - * @param len - * @return - */ -int32_t qIsBuiltinFunction(const char* name, int32_t len, bool* scalarFunction); - bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId); -bool qIsAggregateFunction(const char* functionName); -bool qIsSelectivityFunction(const char* functionName); - tExprNode* exprTreeFromBinary(const void* data, size_t size); void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 2d1808789cc6d0c6ec6c8367d0e9fc23ac952271..ded12620d61cdde001ed0b2e37c32866fd4215a0 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -23,6 +23,7 @@ extern "C" { #include "querynodes.h" typedef struct SDatabaseOptions { + ENodeType type; int32_t numOfBlocks; int32_t cacheBlockSize; int8_t cachelast; @@ -46,7 +47,7 @@ typedef struct SCreateDatabaseStmt { ENodeType type; char dbName[TSDB_DB_NAME_LEN]; bool ignoreExists; - SDatabaseOptions options; + SDatabaseOptions* pOptions; } SCreateDatabaseStmt; typedef struct SUseDatabaseStmt { @@ -60,7 +61,14 @@ typedef struct SDropDatabaseStmt { bool ignoreNotExists; } SDropDatabaseStmt; +typedef struct SAlterDatabaseStmt { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; + SDatabaseOptions* pOptions; +} SAlterDatabaseStmt; + typedef struct STableOptions { + ENodeType type; int32_t keep; int32_t ttl; char comments[TSDB_STB_COMMENT_LEN]; @@ -81,7 +89,7 @@ typedef struct SCreateTableStmt { bool ignoreExists; SNodeList* pCols; SNodeList* pTags; - STableOptions options; + STableOptions* pOptions; } SCreateTableStmt; typedef struct SCreateSubTableClause { @@ -119,6 +127,18 @@ typedef struct SDropSuperTableStmt { bool ignoreNotExists; } SDropSuperTableStmt; +typedef struct SAlterTableStmt { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + int8_t alterType; + char colName[TSDB_COL_NAME_LEN]; + char newColName[TSDB_COL_NAME_LEN]; + STableOptions* pOptions; + SDataType dataType; + SValueNode* pVal; +} SAlterTableStmt; + typedef struct SCreateUserStmt { ENodeType type; char useName[TSDB_USER_LEN]; @@ -150,12 +170,77 @@ typedef struct SDropDnodeStmt { int32_t port; } SDropDnodeStmt; +typedef struct SAlterDnodeStmt { + ENodeType type; + int32_t dnodeId; + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; +} SAlterDnodeStmt; + typedef struct SShowStmt { ENodeType type; SNode* pDbName; // SValueNode SNode* pTbNamePattern; // SValueNode } SShowStmt; +typedef enum EIndexType { + INDEX_TYPE_SMA = 1, + INDEX_TYPE_FULLTEXT +} EIndexType; + +typedef struct SIndexOptions { + ENodeType type; + SNodeList* pFuncs; + SNode* pInterval; + SNode* pOffset; + SNode* pSliding; +} SIndexOptions; + +typedef struct SCreateIndexStmt { + ENodeType type; + EIndexType indexType; + char indexName[TSDB_INDEX_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + SNodeList* pCols; + SIndexOptions* pOptions; +} SCreateIndexStmt; + +typedef struct SDropIndexStmt { + ENodeType type; + char indexName[TSDB_INDEX_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; +} SDropIndexStmt; + +typedef struct SCreateQnodeStmt { + ENodeType type; + int32_t dnodeId; +} SCreateQnodeStmt; + +typedef struct SDropQnodeStmt { + ENodeType type; + int32_t dnodeId; +} SDropQnodeStmt; + +typedef struct SCreateTopicStmt { + ENodeType type; + char topicName[TSDB_TABLE_NAME_LEN]; + char subscribeDbName[TSDB_DB_NAME_LEN]; + bool ignoreExists; + SNode* pQuery; +} SCreateTopicStmt; + +typedef struct SDropTopicStmt { + ENodeType type; + char topicName[TSDB_TABLE_NAME_LEN]; + bool ignoreNotExists; +} SDropTopicStmt; + +typedef struct SAlterLocalStmt { + ENodeType type; + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; +} SAlterLocalStmt; + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 620073dce9a687d9398692d6d4c4383e0a92a8ba..1ab8f2b48c6fd98a1f41ae0e5f6ae846ddb380d9 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -67,6 +67,9 @@ typedef enum ENodeType { QUERY_NODE_SLOT_DESC, QUERY_NODE_COLUMN_DEF, QUERY_NODE_DOWNSTREAM_SOURCE, + QUERY_NODE_DATABASE_OPTIONS, + QUERY_NODE_TABLE_OPTIONS, + QUERY_NODE_INDEX_OPTIONS, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, @@ -74,18 +77,28 @@ typedef enum ENodeType { QUERY_NODE_VNODE_MODIF_STMT, QUERY_NODE_CREATE_DATABASE_STMT, QUERY_NODE_DROP_DATABASE_STMT, + QUERY_NODE_ALTER_DATABASE_STMT, QUERY_NODE_CREATE_TABLE_STMT, QUERY_NODE_CREATE_SUBTABLE_CLAUSE, QUERY_NODE_CREATE_MULTI_TABLE_STMT, QUERY_NODE_DROP_TABLE_CLAUSE, QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT, + QUERY_NODE_ALTER_TABLE_STMT, QUERY_NODE_CREATE_USER_STMT, QUERY_NODE_ALTER_USER_STMT, QUERY_NODE_DROP_USER_STMT, QUERY_NODE_USE_DATABASE_STMT, QUERY_NODE_CREATE_DNODE_STMT, QUERY_NODE_DROP_DNODE_STMT, + QUERY_NODE_ALTER_DNODE_STMT, + QUERY_NODE_CREATE_INDEX_STMT, + QUERY_NODE_DROP_INDEX_STMT, + QUERY_NODE_CREATE_QNODE_STMT, + QUERY_NODE_DROP_QNODE_STMT, + QUERY_NODE_CREATE_TOPIC_STMT, + QUERY_NODE_DROP_TOPIC_STMT, + QUERY_NODE_ALTER_LOCAL_STMT, QUERY_NODE_SHOW_DATABASES_STMT, QUERY_NODE_SHOW_TABLES_STMT, QUERY_NODE_SHOW_STABLES_STMT, @@ -157,6 +170,7 @@ SNodeList* nodesMakeList(); int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); +int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); @@ -190,6 +204,9 @@ const char* nodesNodeName(ENodeType type); int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToNode(const char* pStr, SNode** pNode); +int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen); +int32_t nodesStringToList(const char* pStr, SNodeList** pList); + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 0d4d8423b114d67249f5773ee1f2f1faec372de4..77c59a38c4444408096d33aa68c49bcd611cbd66 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -26,7 +26,6 @@ extern "C" { typedef struct SLogicNode { ENodeType type; - int32_t id; SNodeList* pTargets; // SColumnNode SNode* pConditions; SNodeList* pChildren; @@ -69,7 +68,7 @@ typedef struct SProjectLogicNode { } SProjectLogicNode; typedef struct SVnodeModifLogicNode { - SLogicNode node;; + SLogicNode node; int32_t msgType; SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; @@ -93,6 +92,8 @@ typedef struct SWindowLogicNode { int64_t interval; int64_t offset; int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; SFillNode* pFill; } SWindowLogicNode; @@ -153,7 +154,7 @@ typedef struct SPhysiNode { } SPhysiNode; typedef struct SScanPhysiNode { - SPhysiNode node; + SPhysiNode node; SNodeList* pScanCols; uint64_t uid; // unique id of the table int8_t tableType; @@ -164,6 +165,7 @@ typedef struct SScanPhysiNode { } SScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode; +typedef SScanPhysiNode SStreamScanPhysiNode; typedef struct SSystemTableScanPhysiNode { SScanPhysiNode scan; @@ -207,7 +209,7 @@ typedef struct SDownstreamSourceNode { typedef struct SExchangePhysiNode { SPhysiNode node; - int32_t srcGroupId; // group id of datasource suplans + int32_t srcGroupId; // group id of datasource suplans SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; @@ -215,9 +217,11 @@ typedef struct SIntervalPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; - int64_t interval; - int64_t offset; - int64_t sliding; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; SFillNode* pFill; } SIntervalPhysiNode; @@ -253,7 +257,7 @@ typedef struct SSubplan { } SSubplan; typedef struct SQueryPlan { - ENodeType type;; + ENodeType type; uint64_t queryId; int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index f279b6c66386d8861309d48b94836390ba0ef50b..b6a7892f18b2e131173d00c0f11b3e168327ebdb 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -23,6 +23,10 @@ extern "C" { #include "nodes.h" #include "tmsg.h" +#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) +#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema))) +#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo))) + typedef struct SRawExprNode { ENodeType nodeType; char* p; @@ -82,6 +86,7 @@ typedef struct SValueNode { double d; char* p; } datum; + char unit; } SValueNode; typedef struct SOperatorNode { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index bf0c963059cf189a011a3cf0f10c4d53fb14999e..74a15e2d1854890d324bf92447af9fc958abdc37 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -26,6 +26,7 @@ typedef struct SParseContext { uint64_t requestId; int32_t acctId; const char *db; + bool topicQuery; void *pTransporter; SEpSet mgmtEpSet; const char *pSql; // sql string @@ -51,6 +52,7 @@ typedef struct SQuery { SSchema* pResSchema; SCmdMsgInfo* pCmdMsg; int32_t msgType; + bool streamQuery; } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index fe1b2d27f944c3613201e2dc756bbb6273da0402..38b30ec01e9d051100c5759a3f8a1a2af5d88bba 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -27,6 +27,8 @@ typedef struct SPlanContext { int32_t acctId; SEpSet mgmtEpSet; SNode* pAstRoot; + bool topicQuery; + bool streamQuery; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index affa265d53b6dcdd60353bc826be4323e5c0d605..329329206b2a1c07e5e28fcaa227b4acabdef3e5 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -49,6 +49,10 @@ typedef struct STableComInfo { int32_t rowSize; // row size of the schema } STableComInfo; +typedef struct SIndexMeta { + +} SIndexMeta; + /* * ASSERT(sizeof(SCTableMeta) == 24) * ASSERT(tableType == TSDB_CHILD_TABLE) diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index dd3103edf1baf4f161f7dd7781934c818a919059..8954cbaa0ec1f583b0524413ce9e73eb189e0d16 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -49,10 +49,10 @@ typedef struct { } SQWorkerStat; typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); -typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *); +typedef int32_t (*sendReqFp)(void *, struct SEpSet *, struct SRpcMsg *); int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, - putReqToQueryQFp fp1, sendReqToDnodeFp fp2); + putReqToQueryQFp fp1, sendReqFp fp2); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 1b41da33bbbcfa60567837c9a886533b42062b54..1dc154ce484e1a923852f877bdae4ec37d62b1b4 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -198,6 +198,16 @@ void tfsBasename(const STfsFile *pFile, char *dest); */ void tfsDirname(const STfsFile *pFile, char *dest); +/** + * @brief Get the absolute file name of rname. + * + * @param pTfs + * @param diskId + * @param rname relative file name + * @param aname absolute file name + */ +void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname); + /** * @brief Remove file in tfs. * diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 8dfd736df6f3984f823b73d2703f154f68803afc..02029a996cd315bc5520da7041ccecee82f52045 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -29,7 +29,6 @@ extern "C" { extern int tsRpcHeadSize; - typedef struct SRpcConnInfo { uint32_t clientIp; uint16_t clientPort; @@ -42,10 +41,20 @@ typedef struct SRpcMsg { void * pCont; int contLen; int32_t code; - void * handle; // rpc handle returned to app - void * ahandle; // app handle set by client + void * handle; // rpc handle returned to app + void * ahandle; // app handle set by client + int noResp; // has response or not(default 0, 0: resp, 1: no resp); + int persistHandle; // persist handle or not + } SRpcMsg; +typedef struct { + char user[TSDB_USER_LEN]; + SRpcMsg rpcMsg; + int32_t rspLen; + void *pRsp; + void *pNode; +} SNodeMsg; typedef struct SRpcInit { uint16_t localPort; // local port @@ -68,16 +77,19 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); - // call back to keep conn or not - bool (*pfp)(void *parent, tmsg_t msgType); - - // to support Send messages multiple times on a link - // - void* (*mfp)(void *parent, tmsg_t msgType); - void *parent; } SRpcInit; +typedef struct { + void * val; + int32_t len; + void (*free)(void *arg); +} SRpcCtxVal; + +typedef struct { + SHashObj *args; +} SRpcCtx; + int32_t rpcInit(); void rpcCleanup(); void * rpcOpen(const SRpcInit *pRpc); @@ -86,16 +98,17 @@ void * rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void * rpcReallocCont(void *ptr, int contLen); void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); -void rpcSendResponse(const SRpcMsg *pMsg); -void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); -int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -int rpcReportProgress(void *pConn, char *pCont, int contLen); -void rpcCancelRequest(int64_t rid); - +void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); + +void rpcSendResponse(const SRpcMsg *pMsg); +void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); +int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); +void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int rpcReportProgress(void *pConn, char *pCont, int contLen); +void rpcCancelRequest(int64_t rid); +void rpcRegisterBrokenLinkArg(SRpcMsg *msg); // just release client conn to rpc instance, no close sock -void rpcReleaseHandle(void *handle); - +void rpcReleaseHandle(void *handle, int8_t type); // void rpcRefHandle(void *handle, int8_t type); void rpcUnrefHandle(void *handle, int8_t type); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 32fcd95948d7ef1e481f04d484c9e54070c47f4e..7342af4d1844f2a561bd15240f82326d6e20d618 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -127,7 +127,7 @@ typedef struct SWal { int64_t lastRollSeq; // ctl int64_t refId; - pthread_mutex_t mutex; + TdThreadMutex mutex; // path char path[WAL_PATH_LEN]; // reusable write head diff --git a/include/os/os.h b/include/os/os.h index fd272b15a308222cf980b92b56b86780e232b1fa..b05bfab6d02b5bdd8e1c7fe8be8ad39f10feed5b 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -22,11 +22,13 @@ extern "C" { #include #include +#include + +#include #if !defined(WINDOWS) #include #include -#include #include #include #include @@ -34,7 +36,18 @@ extern "C" { #include #include #include +#include +#include +#include +#include +#include +#include + +#if defined(DARWIN) +#else #include +#include +#endif #endif @@ -54,8 +67,6 @@ extern "C" { #include #include #include -#include -#include #include #include @@ -71,6 +82,7 @@ extern "C" { #include "osMath.h" #include "osMemory.h" #include "osRand.h" +#include "osThread.h" #include "osSemaphore.h" #include "osSignal.h" #include "osSleep.h" @@ -78,7 +90,6 @@ extern "C" { #include "osString.h" #include "osSysinfo.h" #include "osSystem.h" -#include "osThread.h" #include "osTime.h" #include "osTimer.h" #include "osTimezone.h" diff --git a/include/os/osDef.h b/include/os/osDef.h index fcc96c19b46eb51da82ea5fcc0d87b7a4b90d742..492df1047c75ee43010f9a10712d05668d598624 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -56,6 +56,7 @@ extern "C" { // specific typedef int (*__compar_fn_t)(const void *, const void *); #define ssize_t int + #define _SSIZE_T_ #define bzero(ptr, size) memset((ptr), 0, (size)) #define strcasecmp _stricmp #define strncasecmp _strnicmp diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 1426ba87f652552c9cf85cc98777654494648cb1..ebf4c360dd7c5c65672d70248b060960517a5ff4 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -45,6 +45,7 @@ extern SDiskSpace tsTempSpace; void osInit(); void osUpdate(); +void osCleanup(); bool osLogSpaceAvailable(); void osSetTimezone(const char *timezone); diff --git a/include/os/osEok.h b/include/os/osEok.h new file mode 100644 index 0000000000000000000000000000000000000000..3ca476f840c1fc35bc796ecb4de67f6e0a156ca9 --- /dev/null +++ b/include/os/osEok.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_EOK_H +#define TDENGINE_OS_EOK_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __APPLE__ + +enum EPOLL_EVENTS + { + EPOLLIN = 0x001, +#define EPOLLIN EPOLLIN + EPOLLPRI = 0x002, +#define EPOLLPRI EPOLLPRI + EPOLLOUT = 0x004, +#define EPOLLOUT EPOLLOUT + EPOLLRDNORM = 0x040, +#define EPOLLRDNORM EPOLLRDNORM + EPOLLRDBAND = 0x080, +#define EPOLLRDBAND EPOLLRDBAND + EPOLLWRNORM = 0x100, +#define EPOLLWRNORM EPOLLWRNORM + EPOLLWRBAND = 0x200, +#define EPOLLWRBAND EPOLLWRBAND + EPOLLMSG = 0x400, +#define EPOLLMSG EPOLLMSG + EPOLLERR = 0x008, +#define EPOLLERR EPOLLERR + EPOLLHUP = 0x010, +#define EPOLLHUP EPOLLHUP + EPOLLRDHUP = 0x2000, +#define EPOLLRDHUP EPOLLRDHUP + EPOLLEXCLUSIVE = 1u << 28, +#define EPOLLEXCLUSIVE EPOLLEXCLUSIVE + EPOLLWAKEUP = 1u << 29, +#define EPOLLWAKEUP EPOLLWAKEUP + EPOLLONESHOT = 1u << 30, +#define EPOLLONESHOT EPOLLONESHOT + EPOLLET = 1u << 31 +#define EPOLLET EPOLLET + }; + +/* Valid opcodes ( "op" parameter ) to issue to epoll_ctl(). */ +#define EPOLL_CTL_ADD 1 /* Add a file descriptor to the interface. */ +#define EPOLL_CTL_DEL 2 /* Remove a file descriptor from the interface. */ +#define EPOLL_CTL_MOD 3 /* Change file descriptor epoll_event structure. */ + + +typedef union epoll_data +{ + void *ptr; + int fd; + uint32_t u32; + uint64_t u64; +} epoll_data_t; + +struct epoll_event +{ + uint32_t events; /* Epoll events */ + epoll_data_t data; /* User data variable */ +}; + +int epoll_create(int size); +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); +int epoll_close(int epfd); + +#endif // __APPLE__ + +#ifdef __cplusplus +} +#endif + +#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + diff --git a/include/os/osFile.h b/include/os/osFile.h index b0e97e7b54902eb9c05f13486b8c44b40e533b9c..508a52267901994da5bb1d8468b7bf48992da357 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,15 +22,6 @@ extern "C" { #include "osSocket.h" -#if defined(WINDOWS) -typedef int32_t FileFd; -typedef SOCKET SocketFd; -#else -typedef int32_t FileFd; -typedef int32_t SocketFd; -#endif - -int64_t taosRead(FileFd fd, void *buf, int64_t count); // If the error is in a third-party library, place this header file under the third-party library header file. #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID @@ -42,6 +33,7 @@ int64_t taosRead(FileFd fd, void *buf, int64_t count); #define close CLOSE_FUNC_TAOS_FORBID #define fclose FCLOSE_FUNC_TAOS_FORBID #define fsync FSYNC_FUNC_TAOS_FORBID + #define getline GETLINE_FUNC_TAOS_FORBID // #define fflush FFLUSH_FUNC_TAOS_FORBID #endif @@ -86,10 +78,6 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); void taosFprintfFile(TdFilePtr pFile, const char *format, ...); -#if defined(WINDOWS) -#define __restrict__ -#endif // WINDOWS - int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); @@ -101,7 +89,7 @@ int64_t taosCopyFile(const char *from, const char *to); int32_t taosRemoveFile(const char *path); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); - + int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 9904170d61f152f6cbc047f52c65d0deb960edb8..594daf1bf38b2fc0ca6ed522006eeca46012dbb1 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -20,7 +20,6 @@ extern "C" { #endif -#include #include #if defined (_TD_DARWIN_64) @@ -38,25 +37,25 @@ extern "C" { #endif #if defined (_TD_DARWIN_64) - #define pthread_rwlock_t pthread_mutex_t - #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL) - #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock) - #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock) - #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock) - #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock) +// #define TdThreadRwlock TdThreadMutex +// #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL) +// #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock) +// #define taosThreadRwlockWrlock(lock) taosThreadMutexLock(lock) +// #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock) +// #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock) - #define pthread_spinlock_t pthread_mutex_t - #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL) - #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock) - #define pthread_spin_lock(lock) pthread_mutex_lock(lock) - #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock) + #define TdThreadSpinlock TdThreadMutex + #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL) + #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock) + #define taosThreadSpinLock(lock) taosThreadMutexLock(lock) + #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock) #endif -bool taosCheckPthreadValid(pthread_t thread); +bool taosCheckPthreadValid(TdThread thread); int64_t taosGetSelfPthreadId(); -int64_t taosGetPthreadId(pthread_t thread); -void taosResetPthread(pthread_t* thread); -bool taosComparePthread(pthread_t first, pthread_t second); +int64_t taosGetPthreadId(TdThread thread); +void taosResetPthread(TdThread* thread); +bool taosComparePthread(TdThread first, TdThread second); int32_t taosGetPId(); int32_t taosGetAppName(char* name, int32_t* len); diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 3faed855bad7fa42e23ceac0133d870501926a45..fb330a9e2fc62cc90f9047ace4488a954a51f50b 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -25,6 +25,8 @@ #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID + #define inet_addr INET_ADDR_FUNC_TAOS_FORBID + #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID #endif #if defined(WINDOWS) @@ -33,8 +35,15 @@ #include #include #else - #include - #include + #include + #include + + #if defined(_TD_DARWIN_64) + #include + #else + #include + #include + #endif #endif #ifdef __cplusplus @@ -49,7 +58,29 @@ extern "C" { #endif #if defined(_TD_DARWIN_64) - #define htobe64 htonll +// #define htobe64 htonll + +# include + +# define htobe16(x) OSSwapHostToBigInt16(x) +# define htole16(x) OSSwapHostToLittleInt16(x) +# define be16toh(x) OSSwapBigToHostInt16(x) +# define le16toh(x) OSSwapLittleToHostInt16(x) + +# define htobe32(x) OSSwapHostToBigInt32(x) +# define htole32(x) OSSwapHostToLittleInt32(x) +# define be32toh(x) OSSwapBigToHostInt32(x) +# define le32toh(x) OSSwapLittleToHostInt32(x) + +# define htobe64(x) OSSwapHostToBigInt64(x) +# define htole64(x) OSSwapHostToLittleInt64(x) +# define be64toh(x) OSSwapBigToHostInt64(x) +# define le64toh(x) OSSwapLittleToHostInt64(x) + +# define __BYTE_ORDER BYTE_ORDER +# define __BIG_ENDIAN BIG_ENDIAN +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __PDP_ENDIAN PDP_ENDIAN #endif #define TAOS_EPOLL_WAIT_TIME 500 diff --git a/include/os/osString.h b/include/os/osString.h index 88160dd69e419450d1ec481a83a28065df99186d..5846d6b7ed458e1434aab51e84f4bd27ea155370 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -20,16 +20,28 @@ extern "C" { #endif +typedef wchar_t TdWchar; +typedef int32_t TdUcs4; + +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID + #define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID + #define iconv ICONV_FUNC_TAOS_FORBID + #define wcwidth WCWIDTH_FUNC_TAOS_FORBID + #define wcswidth WCSWIDTH_FUNC_TAOS_FORBID + #define mbtowc MBTOWC_FUNC_TAOS_FORBID + #define mbstowcs MBSTOWCS_FUNC_TAOS_FORBID + #define wctomb WCTOMB_FUNC_TAOS_FORBID + #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID + #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID + #define wchar_t WCHAR_T_TYPE_TAOS_FORBID +#endif + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #define tstrdup(str) _strdup(str) - #define tstrndup(str, size) _strndup(str, size) - int32_t tgetline(char **lineptr, size_t *n, FILE *stream); - int32_t twcslen(const wchar_t *wcs); #else #define tstrdup(str) strdup(str) - #define tstrndup(str, size) strndup(str, size) - #define tgetline(lineptr, n, stream) getline(lineptr, n, stream) - #define twcslen wcslen #endif #define tstrncpy(dst, src, size) \ @@ -38,14 +50,22 @@ extern "C" { (dst)[(size)-1] = 0; \ } while (0) +int32_t taosUcs4len(TdUcs4 *ucs4); int64_t taosStr2int64(const char *str); -// USE_LIBICONV -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); -bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); -int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); +int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); +bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); +int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); +TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); +int32_t taosWcharWidth(TdWchar wchar); +int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size); +int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size); +int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size); +int32_t taosWcharToMb(char *pStr, TdWchar wchar); +int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size); + #ifdef __cplusplus } #endif diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index d7cf05a59464a867f09f046e44dd6c8face37dcc..e1cc6f78299f9638583f277cd99ad94b640661d4 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -52,7 +52,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen); char *taosGetCmdlineByPID(int32_t pid); void taosSetCoreDump(bool enable); -#if defined(WINDOWS) +#if !defined(LINUX) #define _UTSNAME_LENGTH 65 #define _UTSNAME_MACHINE_LENGTH _UTSNAME_LENGTH diff --git a/include/os/osThread.h b/include/os/osThread.h index cccc13755d3a33f0ecaa1173df391228d3c90da9..504814c1c600258ff8e250f82b72a53cf34364fc 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -22,6 +22,94 @@ extern "C" { #endif +typedef pthread_t TdThread; +typedef pthread_spinlock_t TdThreadSpinlock; +typedef pthread_mutex_t TdThreadMutex; +typedef pthread_mutexattr_t TdThreadMutexAttr; +typedef pthread_rwlock_t TdThreadRwlock; +typedef pthread_attr_t TdThreadAttr; +typedef pthread_once_t TdThreadOnce; +typedef pthread_rwlockattr_t TdThreadRwlockAttr; +typedef pthread_cond_t TdThreadCond; +typedef pthread_condattr_t TdThreadCondAttr; + +#define taosThreadCleanupPush pthread_cleanup_push +#define taosThreadCleanupPop pthread_cleanup_pop + +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID + #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID + #define pthread_mutex_t PTHREAD_MUTEX_T_TYPE_TAOS_FORBID + #define pthread_mutexattr_t PTHREAD_MUTEXATTR_T_TYPE_TAOS_FORBID + #define pthread_rwlock_t PTHREAD_RWLOCK_T_TYPE_TAOS_FORBID + #define pthread_attr_t PTHREAD_ATTR_T_TYPE_TAOS_FORBID + #define pthread_once_t PTHREAD_ONCE_T_TYPE_TAOS_FORBID + #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID + #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID + #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID + #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID + #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID + #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID + #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID + #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID + #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID + #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID + #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID + #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID + #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID + #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID + #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID + #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID + #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID + #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID + #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID + #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID + #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID + #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID + #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID + #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID + #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID + #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID + #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID + #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID + #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID + #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID +#endif + +int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared); +int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr); +int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock); +int32_t taosThreadMutexDestroy(TdThreadMutex * mutex); +int32_t taosThreadSpinLock(TdThreadSpinlock *lock); +int32_t taosThreadMutexLock(TdThreadMutex *mutex); +int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock); +int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock); +int32_t taosThreadMutexUnlock(TdThreadMutex *mutex); +int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock); +int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock); +void taosThreadTestCancel(void); +int32_t taosThreadAttrInit(TdThreadAttr *attr); +int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg); +int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void)); +int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState); +int32_t taosThreadAttrDestroy(TdThreadAttr *attr); +int32_t taosThreadJoin(TdThread thread, void **pValue); +int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr); +int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock); +int32_t taosThreadCondSignal(TdThreadCond *cond); +int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr); +int32_t taosThreadCondBroadcast(TdThreadCond *cond); +int32_t taosThreadCondDestroy(TdThreadCond *cond); +int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex); +TdThread taosThreadSelf(void); +int32_t taosThreadEqual(TdThread t1, TdThread t2); +int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset); +int32_t taosThreadCancel(TdThread thread); +int32_t taosThreadKill(TdThread thread, int sig); #ifdef __cplusplus } #endif diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index ff015ef0b1221eeba361818f5e010bd7f43fb0fd..fa0e70f209f6b3deee370196d877086459d7ce2e 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -20,6 +20,11 @@ extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define tzset TZSET_FUNC_TAOS_FORBID +#endif + void taosGetSystemTimezone(char *outTimezone); void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 5eb8190136ade13f570ee6e07747e95a13b9f35a..9e1e1c53dca9146249b72d9c83eb59862616dd04 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -75,6 +75,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) #define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) +#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x010E) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) @@ -277,34 +278,14 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) #define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0402) -#define TSDB_CODE_DND_DNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0410) -#define TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0411) -#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0420) -#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0421) -#define TSDB_CODE_DND_MNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0422) -#define TSDB_CODE_DND_MNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0423) -#define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0424) -#define TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0430) -#define TSDB_CODE_DND_QNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0431) -#define TSDB_CODE_DND_QNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0432) -#define TSDB_CODE_DND_QNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0433) -#define TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0434) -#define TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0440) -#define TSDB_CODE_DND_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0441) -#define TSDB_CODE_DND_SNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0442) -#define TSDB_CODE_DND_SNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0443) -#define TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0444) -#define TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0450) -#define TSDB_CODE_DND_BNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0451) -#define TSDB_CODE_DND_BNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0452) -#define TSDB_CODE_DND_BNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0453) -#define TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0454) -#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0460) -#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0461) -#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0462) -#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0463) -#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0464) -#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0465) +#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) +#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0404) +#define TSDB_CODE_NODE_PARSE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0405) +#define TSDB_CODE_NODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0406) +#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0410) +#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) +#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0412) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0413) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) @@ -356,6 +337,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x0617) #define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0618) +#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0619) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) @@ -485,6 +467,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611) #define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612) #define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613) +#define TSDB_CODE_PAR_EXPRIE_STATEMENT TAOS_DEF_ERROR_CODE(0, 0x2614) #ifdef __cplusplus } diff --git a/include/util/tcompare.h b/include/util/tcompare.h index 4c80eeb4f601034852acebd52b2e18b74c52d424..cc9e8ae4641138be528830e17467dab7897f0166 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -46,7 +46,7 @@ typedef struct SPatternCompareInfo { int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo); -int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); +int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo); int32_t taosArrayCompareString(const void *a, const void *b); diff --git a/include/util/tdef.h b/include/util/tdef.h index 057725b1ff395e5042779e3de7db18717e94e67a..7a3e26aa15a42cb3ddccea0b6673d2cdc774bf97 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -41,7 +41,7 @@ extern const int32_t TYPE_BYTES[15]; #define DOUBLE_BYTES sizeof(double) #define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) #define TSDB_KEYSIZE sizeof(TSKEY) -#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define TSDB_NCHAR_SIZE sizeof(TdUcs4) // NULL definition #define TSDB_DATA_BOOL_NULL 0x02 @@ -345,19 +345,19 @@ typedef enum ELogicConditionType { #define TSDB_MAX_DB_QUORUM_OPTION 2 #define TSDB_DEFAULT_DB_QUORUM_OPTION 1 -#define TSDB_MIN_DB_TTL_OPTION 1 -#define TSDB_DEFAULT_DB_TTL_OPTION 0 +#define TSDB_MIN_DB_TTL_OPTION 1 +#define TSDB_DEFAULT_DB_TTL_OPTION 0 -#define TSDB_MIN_DB_SINGLE_STABLE_OPTION 0 -#define TSDB_MAX_DB_SINGLE_STABLE_OPTION 1 -#define TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION 0 +#define TSDB_MIN_DB_SINGLE_STABLE_OPTION 0 +#define TSDB_MAX_DB_SINGLE_STABLE_OPTION 1 +#define TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION 0 -#define TSDB_MIN_DB_STREAM_MODE_OPTION 0 -#define TSDB_MAX_DB_STREAM_MODE_OPTION 1 -#define TSDB_DEFAULT_DB_STREAM_MODE_OPTION 0 +#define TSDB_MIN_DB_STREAM_MODE_OPTION 0 +#define TSDB_MAX_DB_STREAM_MODE_OPTION 1 +#define TSDB_DEFAULT_DB_STREAM_MODE_OPTION 0 -#define TSDB_MAX_JOIN_TABLE_NUM 10 -#define TSDB_MAX_UNION_CLAUSE 5 +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 #define TSDB_MIN_DB_UPDATE 0 #define TSDB_MAX_DB_UPDATE 2 @@ -445,6 +445,14 @@ typedef struct { #define TMQ_SEPARATOR ':' +#define SND_UNIQUE_THREAD_NUM 2 +#define SND_SHARED_THREAD_NUM 2 + +enum { + SND_WORKER_TYPE__SHARED = 1, + SND_WORKER_TYPE__UNIQUE, +}; + #ifdef __cplusplus } #endif diff --git a/include/util/tencode.h b/include/util/tencode.h index c058eebb5016141dc102aeed01621b7f4cb368fc..7e1b624dfb5095536a8c5f170b80af38afc68d52 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -75,7 +75,7 @@ typedef struct { #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl)) +#define TCODER_MALLOC(PTR, TYPE, SIZE, CODER) TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl)) void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderClear(SCoder* pCoder); @@ -402,10 +402,33 @@ static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val) { return 0; } +static FORCE_INLINE int32_t tDecodeBinaryAlloc(SCoder* pDecoder, void** val, uint64_t* len) { + if (tDecodeU64v(pDecoder, len) < 0) return -1; + + if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; + *val = malloc(*len); + if (*val == NULL) return -1; + memcpy(*val, TD_CODER_CURRENT(pDecoder), *len); + + TD_CODER_MOVE_POS(pDecoder, *len); + return 0; +} + +static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SCoder* pDecoder, char** val, uint64_t* len) { + if (tDecodeBinaryAlloc(pDecoder, (void**)val, len) < 0) return -1; + (*len) -= 1; + return 0; +} + +static FORCE_INLINE int32_t tDecodeCStrAlloc(SCoder* pDecoder, char** val) { + uint64_t len; + return tDecodeCStrAndLenAlloc(pDecoder, val, &len); +} + static FORCE_INLINE bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); } #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_ENCODE_H_*/ \ No newline at end of file +#endif /*_TD_UTIL_ENCODE_H_*/ diff --git a/include/util/tfreelist.h b/include/util/tfreelist.h index c1913ebaf95866218db3d6b151a903f040e2175e..0a507aeec9b35951e7e646559cf058d5cb05b69a 100644 --- a/include/util/tfreelist.h +++ b/include/util/tfreelist.h @@ -29,15 +29,17 @@ struct SFreeListNode { typedef TD_SLIST(SFreeListNode) SFreeList; -#define TFL_MALLOC(SIZE, LIST) \ - ({ \ +#define TFL_MALLOC(PTR, TYPE, SIZE, LIST) \ + do { \ void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \ if (ptr) { \ TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ ptr = ((struct SFreeListNode *)ptr)->payload; \ + (PTR) = (TYPE)(ptr); \ + }else{ \ + (PTR) = NULL; \ } \ - ptr; \ - }) + }while(0); #define tFreeListInit(pFL) TD_SLIST_INIT(pFL) diff --git a/include/util/tidpool.h b/include/util/tidpool.h index 8596b439e3f24d684bbabc4c7364f609119d224d..c97f4c5f388ee48f3f86f50b82665f6479c5bb9b 100644 --- a/include/util/tidpool.h +++ b/include/util/tidpool.h @@ -27,7 +27,7 @@ typedef struct { int32_t numOfFree; int32_t freeSlot; bool *freeList; - pthread_mutex_t mutex; + TdThreadMutex mutex; } id_pool_t; void *taosInitIdPool(int32_t maxId); diff --git a/include/util/tjson.h b/include/util/tjson.h index 335ff0d4bada0ca1a4cdb89081171b915bf02664..6b2221f704c768e494ba797913ede2deb5f6a7f4 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -22,9 +22,18 @@ extern "C" { #endif +#define tjsonGetNumberValue(pJson, pName, val) \ + ({ \ + uint64_t _tmp = 0; \ + int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \ + val = _tmp; \ + _code; \ + }) + typedef void SJson; SJson* tjsonCreateObject(); +SJson* tjsonCreateArray(); void tjsonDelete(SJson* pJson); SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName); @@ -43,6 +52,7 @@ int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal); int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal); int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal); int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal); +int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal); int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal); int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal); int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal); @@ -59,6 +69,7 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* typedef int32_t (*FToObject)(const SJson* pJson, void* pObj); int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj); +int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize); int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize); char* tjsonToString(const SJson* pJson); diff --git a/include/util/tprocess.h b/include/util/tprocess.h new file mode 100644 index 0000000000000000000000000000000000000000..4ce536fd968198ef5be19e888607e647e057853c --- /dev/null +++ b/include/util/tprocess.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_PROCESS_H_ +#define _TD_UTIL_PROCESS_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SProcQueue SProcQueue; +typedef struct SProcObj SProcObj; +typedef void *(*ProcMallocFp)(int32_t contLen); +typedef void *(*ProcFreeFp)(void *pCont); +typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); + +typedef struct { + int32_t childQueueSize; + ProcConsumeFp childConsumeFp; + ProcMallocFp childMallocHeadFp; + ProcFreeFp childFreeHeadFp; + ProcMallocFp childMallocBodyFp; + ProcFreeFp childFreeBodyFp; + int32_t parentQueueSize; + ProcConsumeFp parentConsumeFp; + ProcMallocFp parentdMallocHeadFp; + ProcFreeFp parentFreeHeadFp; + ProcMallocFp parentMallocBodyFp; + ProcFreeFp parentFreeBodyFp; + bool testFlag; + void *pParent; + const char *name; +} SProcCfg; + +SProcObj *taosProcInit(const SProcCfg *pCfg); +void taosProcCleanup(SProcObj *pProc); +int32_t taosProcRun(SProcObj *pProc); +void taosProcStop(SProcObj *pProc); +bool taosProcIsChild(SProcObj *pProc); + +int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); +int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_PROCESS_H_*/ diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index 64cab08cfe87d430fded815878a5cad6083a4d81..a2382ad541e104430ad7334b458a8d50a5fb5284 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -57,7 +57,7 @@ typedef struct SSkipListNode { * @date 2017/11/12 * the simple version of skip list. * - * for multi-thread safe purpose, we employ pthread_rwlock_t to guarantee to generate + * for multi-thread safe purpose, we employ TdThreadRwlock to guarantee to generate * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance. * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of * this simple version in a multi-thread environment, to achieve higher performance of read/write operations. @@ -106,7 +106,7 @@ typedef struct SSkipList { uint32_t seed; __compar_fn_t comparFn; __sl_key_fn_t keyFn; - pthread_rwlock_t *lock; + TdThreadRwlock *lock; uint16_t len; uint8_t maxLevel; uint8_t flags; diff --git a/include/util/tthread.h b/include/util/tthread.h index 49412069445d0f1cb89631683daa6411ca88879d..2215add06262ea63cbeb447f505c3a9e2bc134de 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -22,9 +22,11 @@ extern "C" { #endif -pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param); -bool taosDestoryThread(pthread_t* pthread); -bool taosThreadRunning(pthread_t* pthread); +TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param); +bool taosDestoryThread(TdThread* pthread); +bool taosThreadRunning(TdThread* pthread); + +typedef void *(*ThreadFp)(void *param); #ifdef __cplusplus } diff --git a/include/util/tuuid.h b/include/util/tuuid.h new file mode 100644 index 0000000000000000000000000000000000000000..315c2ad4971372d1e1e279f2e79429d8b26dbe3d --- /dev/null +++ b/include/util/tuuid.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" +#include "taoserror.h" +#include "thash.h" + +/** + * Generate an non-negative signed 32bit id + *+------------+-----+-----------+---------------+ + *| uid|localIp| PId | timestamp | serial number | + *+------------+-----+-----------+---------------+ + *| 6bit |6bit | 12bit | 8bit | + *+------------+-----+-----------+---------------+ + * @return + */ +int32_t tGenIdPI32(void); + +/** + * Generate an non-negative signed 64bit id + *+------------+-----+-----------+---------------+ + *| uid|localIp| PId | timestamp | serial number | + *+------------+-----+-----------+---------------+ + *| 12bit |12bit|24bit |16bit | + *+------------+-----+-----------+---------------+ + * @return + */ +int64_t tGenIdPI64(void); diff --git a/include/util/tworker.h b/include/util/tworker.h index e6f6bc077c69f05498df2536c7de55a488dd8f93..dabd1ac9b379b467cbe7c2f3e6b19895ed02db88 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -27,7 +27,7 @@ typedef struct SWWorkerPool SWWorkerPool; typedef struct SQWorker { int32_t id; // worker ID - pthread_t thread; // thread + TdThread thread; // thread SQWorkerPool *pool; } SQWorker, SFWorker; @@ -38,12 +38,12 @@ typedef struct SQWorkerPool { STaosQset *qset; const char *name; SQWorker *workers; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SQWorkerPool, SFWorkerPool; typedef struct SWWorker { int32_t id; // worker id - pthread_t thread; // thread + TdThread thread; // thread STaosQall *qall; STaosQset *qset; // queue set SWWorkerPool *pool; @@ -54,7 +54,7 @@ typedef struct SWWorkerPool { int32_t nextId; // from 0 to max-1, cyclic const char *name; SWWorker *workers; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SWWorkerPool; int32_t tQWorkerInit(SQWorkerPool *pool); diff --git a/include/util/types.h b/include/util/types.h index f7a535c965fd6deac8dc2eb1d486c072a0e867e6..981c457fc10598d7109c44845939364c428f7a66 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -82,7 +82,7 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 #define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] -#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) +#define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) typedef int32_t VarDataOffsetT; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ca07f75de05304c455beca5760b097ccfd4b2b86..24be465313206beec1d2935c5714dc114c38ecdc 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "parser.h" +#include "planner.h" #include "query.h" #include "taos.h" #include "tcommon.h" @@ -75,8 +76,8 @@ typedef struct { int8_t inited; // ctl int8_t threadStop; - pthread_t thread; - pthread_mutex_t lock; // used when app init and cleanup + TdThread thread; + TdThreadMutex lock; // used when app init and cleanup SArray* appHbMgrs; // SArray one for each cluster FHbReqHandle reqHandle[HEARTBEAT_TYPE_MAX]; FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; @@ -123,7 +124,7 @@ typedef struct SAppInfo { int32_t pid; int32_t numOfThreads; SHashObj* pInstMap; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SAppInfo; typedef struct STscObj { @@ -135,7 +136,7 @@ typedef struct STscObj { uint32_t connId; int32_t connType; uint64_t id; // ref ID returned by taosAddRef - pthread_mutex_t mutex; // used to protect the operation on db + TdThreadMutex mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; } STscObj; @@ -177,6 +178,7 @@ typedef struct SRequestObj { uint64_t requestId; int32_t type; // request type STscObj* pTscObj; + char* pDb; char* sqlstr; // sql string int32_t sqlLen; int64_t self; @@ -229,7 +231,8 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); -int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery); +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery); +int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); // --- heartbeat // global, called by mgmt diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 525c5f9fb88503c9237cf6db24701fcf3ee03a23..07f925bce8e5f91186bdff0fd23b6450a3730af0 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -33,7 +33,7 @@ SAppInfo appInfo; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; +static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj *pRequest) { @@ -90,7 +90,6 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; - rpcInit.pfp = persistConnForSpecificMsg; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; @@ -115,7 +114,7 @@ void destroyTscObj(void *pObj) { hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); - pthread_mutex_destroy(&pTscObj->mutex); + taosThreadMutexDestroy(&pTscObj->mutex); tfree(pTscObj); } @@ -134,7 +133,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI tstrncpy(pObj->db, db, tListLen(pObj->db)); } - pthread_mutex_init(&pObj->mutex, NULL); + taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); tscDebug("connObj created, 0x%" PRIx64, pObj->id); @@ -150,6 +149,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty return NULL; } + pRequest->pDb = getDbOfConnection(pObj); pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampMs(); @@ -180,6 +180,7 @@ static void doDestroyRequest(void *p) { tfree(pRequest->msgBuf); tfree(pRequest->sqlstr); tfree(pRequest->pInfo); + tfree(pRequest->pDb); doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryPlan(pRequest->body.pDag); @@ -241,7 +242,7 @@ void taos_init_imp(void) { // transDestroyBuffer(&conn->readBuf); taosGetAppName(appInfo.appName, NULL); - pthread_mutex_init(&appInfo.mutex, NULL); + taosThreadMutexInit(&appInfo.mutex, NULL); appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); @@ -250,7 +251,7 @@ void taos_init_imp(void) { } int taos_init() { - pthread_once(&tscinit, taos_init_imp); + taosThreadOnce(&tscinit, taos_init_imp); return tscInitRes; } @@ -506,9 +507,9 @@ static setConfRet taos_set_config_imp(const char *config){ } setConfRet taos_set_config(const char *config){ - pthread_mutex_lock(&setConfMutex); + taosThreadMutexLock(&setConfMutex); setConfRet ret = taos_set_config_imp(config); - pthread_mutex_unlock(&setConfMutex); + taosThreadMutexUnlock(&setConfMutex); return ret; } #endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 71634447196f640ee8fc5d4358876e15000c5534..479281eec642e1e3b12581ab6a5bf5a12dc1fcc6 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -372,7 +372,7 @@ static void *hbThreadFunc(void *param) { break; } - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { @@ -423,7 +423,7 @@ static void *hbThreadFunc(void *param) { atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); taosMsleep(HEARTBEAT_INTERVAL); } @@ -431,15 +431,15 @@ static void *hbThreadFunc(void *param) { } static int32_t hbCreateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); -// if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { +// if (taosThreadCreate(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { // terrno = TAOS_SYSTEM_ERROR(errno); // return -1; // } -// pthread_attr_destroy(&thAttr); +// taosThreadAttrDestroy(&thAttr); return 0; } @@ -492,15 +492,15 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { return NULL; } - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); return pAppHbMgr; } void appHbMgrCleanup(void) { - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { @@ -511,7 +511,7 @@ void appHbMgrCleanup(void) { pTarget->connInfo = NULL; } - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); } int hbMgrInit() { @@ -520,7 +520,7 @@ int hbMgrInit() { if (old == 1) return 0; clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); - pthread_mutex_init(&clientHbMgr.lock, NULL); + taosThreadMutexInit(&clientHbMgr.lock, NULL); // init handle funcs hbMgrInitHandle(); @@ -539,10 +539,10 @@ void hbMgrCleanUp() { int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); if (old == 0) return; - pthread_mutex_lock(&clientHbMgr.lock); + taosThreadMutexLock(&clientHbMgr.lock); appHbMgrCleanup(); taosArrayDestroy(clientHbMgr.appHbMgrs); - pthread_mutex_unlock(&clientHbMgr.lock); + taosThreadMutexUnlock(&clientHbMgr.lock); clientHbMgr.appHbMgrs = NULL; #endif diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 67001856e6ea66065682d5dae2e549430b68bf46..93642f30d92a56e541d38412fb9235fe82ec2e52 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,8 +1,6 @@ #include "clientInt.h" #include "clientLog.h" -#include "parser.h" -#include "planner.h" #include "scheduler.h" #include "tdatablock.h" #include "tdef.h" @@ -97,7 +95,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, char* key = getClusterKey(user, secretEncrypt, ip, port); SAppInstInfo** pInst = NULL; - pthread_mutex_lock(&appInfo.mutex); + taosThreadMutexLock(&appInfo.mutex); pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); SAppInstInfo* p = NULL; @@ -111,7 +109,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, pInst = &p; } - pthread_mutex_unlock(&appInfo.mutex); + taosThreadMutexUnlock(&appInfo.mutex); tfree(key); return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst); @@ -139,13 +137,14 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* return TSDB_CODE_SUCCESS; } -int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) { +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { STscObj* pTscObj = pRequest->pTscObj; SParseContext cxt = { .requestId = pRequest->requestId, .acctId = pTscObj->acctId, - .db = getDbOfConnection(pTscObj), + .db = pRequest->pDb, + .topicQuery = topicQuery, .pSql = pRequest->sqlstr, .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, @@ -156,7 +155,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) { cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); if (code != TSDB_CODE_SUCCESS) { - tfree(cxt.db); return code; } @@ -165,7 +163,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) { setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); } - tfree(cxt.db); return code; } @@ -220,7 +217,6 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } } - int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; @@ -256,7 +252,7 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); + CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return); if (pQuery->directRpc) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); @@ -668,21 +664,21 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t char* getDbOfConnection(STscObj* pObj) { char* p = NULL; - pthread_mutex_lock(&pObj->mutex); + taosThreadMutexLock(&pObj->mutex); size_t len = strlen(pObj->db); if (len > 0) { p = strndup(pObj->db, tListLen(pObj->db)); } - pthread_mutex_unlock(&pObj->mutex); + taosThreadMutexUnlock(&pObj->mutex); return p; } void setConnectionDB(STscObj* pTscObj, const char* db) { assert(db != NULL && pTscObj != NULL); - pthread_mutex_lock(&pTscObj->mutex); + taosThreadMutexLock(&pTscObj->mutex); tstrncpy(pTscObj->db, db, tListLen(pTscObj->db)); - pthread_mutex_unlock(&pTscObj->mutex); + taosThreadMutexUnlock(&pTscObj->mutex); } void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 9534c1164631b1788e6ee6447ca6c0b289e523f0..010f4e6c12e6a24d7882d82ce731daa13a401d32 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -33,7 +33,7 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { setErrno(pRequest, code); free(pMsg->pData); - sem_post(&pRequest->body.rspSem); + tsem_post(&pRequest->body.rspSem); return code; } @@ -42,7 +42,7 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { free(pMsg->pData); setErrno(pRequest, code); - sem_post(&pRequest->body.rspSem); + tsem_post(&pRequest->body.rspSem); return code; } @@ -78,7 +78,7 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->pAppInfo->numOfConns); free(pMsg->pData); - sem_post(&pRequest->body.rspSem); + tsem_post(&pRequest->body.rspSem); return 0; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 846329f0f4d995d657f995e65b510ec4e11d0932..638d8c9c9ae1a152b20ab8a2bac160eae37b50fb 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -459,7 +459,7 @@ void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf-> TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { STscObj* pTscObj = (STscObj*)taos; SRequestObj* pRequest = NULL; - SQuery* pQueryNode = NULL; + SQuery* pQueryNode = NULL; char* pStr = NULL; terrno = TSDB_CODE_SUCCESS; @@ -482,40 +482,25 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i } tscDebug("start to create topic, %s", topicName); -#if 0 - CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); - SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo*)pQueryNode; - pQueryStmtInfo->info.continueQuery = true; + int32_t code = TSDB_CODE_SUCCESS; + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return); // todo check for invalid sql statement and return with error code - SSchema* schema = NULL; - int32_t numOfCols = 0; - CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), - _return); - - pStr = qDagToString(pRequest->body.pDag); - if (pStr == NULL) { - goto _return; - } + CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &pStr, NULL), _return); /*printf("%s\n", pStr);*/ - // The topic should be related to a database that the queried table is belonged to. - SName name = {0}; - char dbName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName); - - tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB); - tNameFromString(&name, topicName, T_NAME_TABLE); + SName name = { .acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T }; + strcpy(name.dbname, pRequest->pDb); + strcpy(name.tname, topicName); SCMCreateTopicReq req = { .igExists = 1, - .physicalPlan = (char*)pStr, + .ast = (char*)pStr, .sql = (char*)sql, - .logicalPlan = (char*)"no logic plan", }; tNameExtractFullName(&name, req.name); @@ -538,7 +523,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); tsem_wait(&pRequest->body.rspSem); -#endif + _return: qDestroyQuery(pQueryNode); /*if (sendInfo != NULL) {*/ @@ -702,6 +687,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); + pRsp->curBlock = 0; + pRsp->curRow = 0; + // TODO: alloc mem + /*pRsp->*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->consumeRsp.numOfTopics == 0) { /*printf("no data\n");*/ @@ -760,9 +749,9 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { goto END; } - // tmq's epoch is monotomically increase, + // tmq's epoch is monotonically increase, // so it's safe to discard any old epoch msg. - // epoch will only increase when received newer epoch ep msg + // Epoch will only increase when received newer epoch ep msg SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); if (head->epoch <= epoch) { @@ -1284,6 +1273,34 @@ const char* tmq_err2str(tmq_resp_err_t err) { return "fail"; } +TAOS_ROW tmq_get_row(tmq_message_t* message) { + SMqPollRsp* rsp = &message->consumeRsp; + while (1) { + if (message->curBlock < taosArrayGetSize(rsp->pBlockData)) { + SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->curBlock); + if (message->curRow < pBlock->info.rows) { + for (int i = 0; i < pBlock->info.numOfCols; i++) { + SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i); + if (colDataIsNull_s(pData, message->curRow)) + message->uData[i] = NULL; + else { + message->uData[i] = colDataGetData(pData, message->curRow); + } + } + message->curRow++; + return message->uData; + } else { + message->curBlock++; + message->curRow = 0; + continue; + } + } + return NULL; + } +} + +char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; } + #if 0 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { tmq_t* pTmq = malloc(sizeof(tmq_t)); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 35c9e963e68e74d8c3c9af63015f521f531e7fe2..fa2a50acd9619b6318c25702b4ac1532c2b64130 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -290,10 +290,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { - ASSERT(pBlock); - - size_t constantCols = (pBlock->pConstantList != NULL)? taosArrayGetSize(pBlock->pConstantList):0; - ASSERT( pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock) + constantCols); + ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock)); return pBlock->info.numOfCols; } @@ -914,7 +911,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn); int64_t p1 = taosGetTimestampUs(); - printf("sort:%ld, rows:%d\n", p1 - p0, pDataBlock->info.rows); + printf("sort:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows); return TSDB_CODE_SUCCESS; } else { // var data type @@ -962,7 +959,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - printf("sort:%ld, create:%ld, assign:%ld, copyback:%ld, rows:%d\n", p1-p0, p2 - p1, p3 - p2, p4-p3, rows); + printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1-p0, p2 - p1, p3 - p2, p4-p3, rows); destroyTupleIndex(index); return TSDB_CODE_SUCCESS; @@ -1067,7 +1064,7 @@ int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { } int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { - + return 0; } int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { @@ -1105,14 +1102,15 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - printf("sort:%ld, create:%ld, assign:%ld, copyback:%ld, rows:%d\n", p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); + printf("sort:%" PRId64 ", create:%" PRId64", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); // destroyTupleIndex(index); + return 0; } -void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol) { +void blockDataClearup(SSDataBlock* pDataBlock) { pDataBlock->info.rows = 0; - if (hasVarCol) { + if (pDataBlock->info.hasVarCol) { for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); @@ -1124,6 +1122,10 @@ void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol) { } int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + if (0 == numOfRows) { + return TSDB_CODE_SUCCESS; + } + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); if (tmp == NULL) { @@ -1144,7 +1146,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo pColumn->nullbitmap = tmp; memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); - + assert(pColumn->info.bytes); tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1189,7 +1191,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) { taosArrayDestroy(pBlock->pDataBlock); tfree(pBlock->pBlockAgg); - tfree(pBlock); + // tfree(pBlock); return NULL; } @@ -1198,7 +1200,9 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = numOfCols; + pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; for(int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; @@ -1213,3 +1217,67 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock)); } + +int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { + int64_t tbUid = pBlock->info.uid; + int16_t numOfCols = pBlock->info.numOfCols; + int16_t hasVarCol = pBlock->info.hasVarCol; + int32_t rows = pBlock->info.rows; + int32_t sz = taosArrayGetSize(pBlock->pDataBlock); + + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, tbUid); + tlen += taosEncodeFixedI16(buf, numOfCols); + tlen += taosEncodeFixedI16(buf, hasVarCol); + tlen += taosEncodeFixedI32(buf, rows); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + tlen += taosEncodeFixedI16(buf, pColData->info.colId); + tlen += taosEncodeFixedI16(buf, pColData->info.type); + tlen += taosEncodeFixedI32(buf, pColData->info.bytes); + + if (IS_VAR_DATA_TYPE(pColData->info.type)) { + tlen += taosEncodeBinary(buf, pColData->varmeta.offset, sizeof(int32_t) * rows); + } else { + tlen += taosEncodeBinary(buf, pColData->nullbitmap, BitmapLen(rows)); + } + + int32_t len = colDataGetLength(pColData, rows); + tlen += taosEncodeFixedI32(buf, len); + + tlen += taosEncodeBinary(buf, pColData->pData, len); + } + return tlen; +} + +void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { + int32_t sz; + + buf = taosDecodeFixedI64(buf, &pBlock->info.uid); + buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols); + buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); + buf = taosDecodeFixedI32(buf, &pBlock->info.rows); + buf = taosDecodeFixedI32(buf, &sz); + pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < sz; i++) { + SColumnInfoData data = {0}; + buf = taosDecodeFixedI16(buf, &data.info.colId); + buf = taosDecodeFixedI16(buf, &data.info.type); + buf = taosDecodeFixedI32(buf, &data.info.bytes); + + if (IS_VAR_DATA_TYPE(data.info.type)) { + buf = taosDecodeBinary(buf, (void**)&data.varmeta.offset, pBlock->info.rows * sizeof(int32_t)); + data.varmeta.length = pBlock->info.rows * sizeof(int32_t); + data.varmeta.allocLen = data.varmeta.length; + } else { + buf = taosDecodeBinary(buf, (void**)&data.nullbitmap, BitmapLen(pBlock->info.rows)); + } + + int32_t len = 0; + buf = taosDecodeFixedI32(buf, &len); + buf = taosDecodeBinary(buf, (void**)&data.pData, len); + taosArrayPush(pBlock->pDataBlock, &data); + } + return (void*)buf; +} \ No newline at end of file diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9a0309e0245a517b452c25a6fff028755f8f7aa9..841824a7c75c4d5c9f2fb858e7690f29d266ca6b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -45,6 +45,7 @@ float tsRatioOfQueryCores = 1.0f; int32_t tsMaxBinaryDisplayWidth = 30; bool tsEnableSlaveQuery = 1; bool tsPrintAuth = 0; +int32_t tsMultiProcess = 0; // monitor bool tsEnableMonitor = 1; @@ -309,7 +310,6 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1; if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1; - if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; if (cfgAddString(pCfg, "version", version, 1) != 0) return -1; if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1; @@ -340,6 +340,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1; if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1; @@ -403,7 +404,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } - tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; + tsNumOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval; tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; @@ -457,6 +458,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval; + tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->i32; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 621c1d43d96e4e198b98eefca7b40ed572a0d54c..d5909e2bb796a29b3f79057261e35ca910846df9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -757,6 +757,8 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { return 0; } +void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); } + int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1467,8 +1469,7 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { return 0; } - -int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq) { +int32_t tSerializeSQnodeListReq(void *buf, int32_t bufLen, SQnodeListReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1499,7 +1500,7 @@ int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) if (tStartEncode(&encoder) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->epSetList); - if (tEncodeI32(&encoder, num) < 0) return -1; + if (tEncodeI32(&encoder, num) < 0) return -1; for (int32_t i = 0; i < num; ++i) { SEpSet *epSet = taosArrayGet(pRsp->epSetList, i); if (tEncodeSEpSet(&encoder, epSet) < 0) return -1; @@ -1995,11 +1996,9 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTopicReq *pReq) { int32_t sqlLen = 0; - int32_t physicalPlanLen = 0; - int32_t logicalPlanLen = 0; + int32_t astLen = 0; if (pReq->sql != NULL) sqlLen = (int32_t)strlen(pReq->sql); - if (pReq->physicalPlan != NULL) physicalPlanLen = (int32_t)strlen(pReq->physicalPlan); - if (pReq->logicalPlan != NULL) logicalPlanLen = (int32_t)strlen(pReq->logicalPlan); + if (pReq->ast != NULL) astLen = (int32_t)strlen(pReq->ast); SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2008,11 +2007,9 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1; - if (tEncodeI32(&encoder, physicalPlanLen) < 0) return -1; - if (tEncodeI32(&encoder, logicalPlanLen) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1; + if (tEncodeI32(&encoder, astLen) < 0) return -1; + if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; tEndEncode(&encoder); @@ -2023,8 +2020,7 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicReq *pReq) { int32_t sqlLen = 0; - int32_t physicalPlanLen = 0; - int32_t logicalPlanLen = 0; + int32_t astLen = 0; SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2033,17 +2029,20 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; - if (tDecodeI32(&decoder, &physicalPlanLen) < 0) return -1; - if (tDecodeI32(&decoder, &logicalPlanLen) < 0) return -1; + if (tDecodeI32(&decoder, &astLen) < 0) return -1; + + if (sqlLen > 0) { + pReq->sql = calloc(1, sqlLen + 1); + if (pReq->sql == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; + } - pReq->sql = calloc(1, sqlLen + 1); - pReq->physicalPlan = calloc(1, physicalPlanLen + 1); - pReq->logicalPlan = calloc(1, logicalPlanLen + 1); - if (pReq->sql == NULL || pReq->physicalPlan == NULL || pReq->logicalPlan == NULL) return -1; + if (astLen > 0) { + pReq->ast = calloc(1, astLen + 1); + if (pReq->ast == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; + } - if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->physicalPlan) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->logicalPlan) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); @@ -2052,8 +2051,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { tfree(pReq->sql); - tfree(pReq->physicalPlan); - tfree(pReq->logicalPlan); + tfree(pReq->ast); } int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) { @@ -2476,7 +2474,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); + TCODER_MALLOC(pReq->offsets, SMqOffset*, pReq->num * sizeof(SMqOffset), decoder); if (pReq->offsets == NULL) return -1; for (int32_t i = 0; i < pReq->num; i++) { tDecodeSMqOffset(decoder, &pReq->offsets[i]); @@ -2496,27 +2494,27 @@ int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pR tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeU64(&encoder, pReq->sId) < 0) return -1; - if (tEncodeI32(&encoder, pReq->epId.nodeId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->sId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->epId.nodeId) < 0) return -1; if (tEncodeU16(&encoder, pReq->epId.ep.port) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->epId.ep.fqdn) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->epId.ep.fqdn) < 0) return -1; if (pReq->taskAction) { int32_t num = taosArrayGetSize(pReq->taskAction); - if (tEncodeI32(&encoder, num) < 0) return -1; + if (tEncodeI32(&encoder, num) < 0) return -1; for (int32_t i = 0; i < num; ++i) { STaskAction *action = taosArrayGet(pReq->taskAction, i); - if (tEncodeU64(&encoder, action->queryId) < 0) return -1; - if (tEncodeU64(&encoder, action->taskId) < 0) return -1; - if (tEncodeI8(&encoder, action->action) < 0) return -1; + if (tEncodeU64(&encoder, action->queryId) < 0) return -1; + if (tEncodeU64(&encoder, action->taskId) < 0) return -1; + if (tEncodeI8(&encoder, action->action) < 0) return -1; } } else { - if (tEncodeI32(&encoder, 0) < 0) return -1; + if (tEncodeI32(&encoder, 0) < 0) return -1; } tEndEncode(&encoder); int32_t tlen = encoder.pos; tCoderClear(&encoder); - + if (buf != NULL) { SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); pHead->vgId = htonl(pReq->header.vgId); @@ -2564,29 +2562,27 @@ int32_t tDeserializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq * void tFreeSSchedulerHbReq(SSchedulerHbReq *pReq) { taosArrayDestroy(pReq->taskAction); } - - int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeU64(&encoder, pRsp->seqId) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->epId.nodeId) < 0) return -1; + if (tEncodeU64(&encoder, pRsp->seqId) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->epId.nodeId) < 0) return -1; if (tEncodeU16(&encoder, pRsp->epId.ep.port) < 0) return -1; - if (tEncodeCStr(&encoder, pRsp->epId.ep.fqdn) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->epId.ep.fqdn) < 0) return -1; if (pRsp->taskStatus) { int32_t num = taosArrayGetSize(pRsp->taskStatus); - if (tEncodeI32(&encoder, num) < 0) return -1; + if (tEncodeI32(&encoder, num) < 0) return -1; for (int32_t i = 0; i < num; ++i) { STaskStatus *status = taosArrayGet(pRsp->taskStatus, i); - if (tEncodeU64(&encoder, status->queryId) < 0) return -1; - if (tEncodeU64(&encoder, status->taskId) < 0) return -1; - if (tEncodeI64(&encoder, status->refId) < 0) return -1; - if (tEncodeI8(&encoder, status->status) < 0) return -1; + if (tEncodeU64(&encoder, status->queryId) < 0) return -1; + if (tEncodeU64(&encoder, status->taskId) < 0) return -1; + if (tEncodeI64(&encoder, status->refId) < 0) return -1; + if (tEncodeI8(&encoder, status->status) < 0) return -1; } } else { - if (tEncodeI32(&encoder, 0) < 0) return -1; + if (tEncodeI32(&encoder, 0) < 0) return -1; } tEndEncode(&encoder); @@ -2757,9 +2753,9 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; - if (tDecodeCStr(&decoder, (const char **)&pReq->sql) < 0) return -1; - if (tDecodeCStr(&decoder, (const char **)&pReq->physicalPlan) < 0) return -1; - if (tDecodeCStr(&decoder, (const char **)&pReq->logicalPlan) < 0) return -1; + if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; + if (tDecodeCStrAlloc(&decoder, &pReq->physicalPlan) < 0) return -1; + if (tDecodeCStrAlloc(&decoder, &pReq->logicalPlan) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); @@ -2771,3 +2767,32 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { tfree(pReq->physicalPlan); tfree(pReq->logicalPlan); } + +int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->level) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +void tFreeSStreamTask(SStreamTask *pTask) { + // TODO + /*free(pTask->qmsg);*/ + /*free(pTask->executor);*/ + /*free(pTask);*/ +} diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 932048edb4ff226f20c6ec7c0d5355327dac91ee..5561eb93c3dd8db15d4eac1f89e3869915a88325 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -217,7 +217,7 @@ int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t name } int32_t tNameSetAcctId(SName* dst, int32_t acctId) { - assert(dst != NULL && acct != NULL); + assert(dst != NULL); dst->acctId = acctId; return 0; } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 861b4dc09334beb2d62e7e0677519c448855a38f..db4bc49425e1835f3451bea18049f4488a1c2b42 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -353,10 +353,10 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i for (int i = 0; i < src2->numOfCols; i++) { SCellVal sVal = {0}; ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) { - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) { - TASSERT(0); - } + if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2) < 0) { + TASSERT(0); + } + if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) { tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints); } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) { diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 464c29d28756b2dd4a42e80f654ed9c7d3d68dd9..5f0a3532268f3222d3852d5b31e7f999fe307ff9 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -845,7 +845,11 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; +#if defined(_TD_DARWIN_64) + int64_t written = taosFSendFile(pDestBuf->pFile->fp, pSrcBuf->pFile->fp, &offset, size); +#else int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size); +#endif if (written == -1 || written != size) { return -1; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index af6152d3f437ba673d9f7969101c6631ffa4eb7c..c6aa1cb81d7a7fe1c48f90f37218a14e63f7cf07 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -199,8 +199,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; - pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); - memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); + pVar->ucs4 = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); + memcpy(pVar->ucs4, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; @@ -343,7 +343,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) { case TSDB_DATA_TYPE_NCHAR: { dst[0] = '\''; - taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); + taosUcs4ToMbs(pVar->ucs4, (taosUcs4len(pVar->ucs4) + 1) * TSDB_NCHAR_SIZE, dst + 1); int32_t len = (int32_t)strlen(dst); dst[len] = '\''; dst[len + 1] = 0; @@ -384,7 +384,7 @@ static FORCE_INLINE int32_t convertToBoolImpl(char *pStr, int32_t len) { } } -static FORCE_INLINE int32_t wcsconvertToBoolImpl(wchar_t *pstr, int32_t len) { +static FORCE_INLINE int32_t wcsconvertToBoolImpl(TdUcs4 *pstr, int32_t len) { if ((wcsncasecmp(pstr, L"true", len) == 0) && (len == 4)) { return TSDB_TRUE; } else if (wcsncasecmp(pstr, L"false", len) == 0 && (len == 5)) { @@ -412,11 +412,11 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { pBuf = realloc(pBuf, newSize + 1); } - taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf); - free(pVariant->wpz); + taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, pBuf); + free(pVariant->ucs4); pBuf[newSize] = 0; } else { - taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest); + taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, *pDest); } } else { @@ -460,8 +460,8 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } if (*pDest == pVariant->pz) { - wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); - bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); + TdUcs4 *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); + bool ret = taosMbsToUcs4(pDst, nLen, pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); if (!ret) { tfree(pWStr); return -1; @@ -469,21 +469,21 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { - free(pVariant->wpz); + free(pVariant->ucs4); } - pVariant->wpz = pWStr; - *pDestSize = twcslen(pVariant->wpz); + pVariant->ucs4 = pWStr; + *pDestSize = taosUcs4len(pVariant->ucs4); // shrink the allocate memory, no need to check here. - char *tmp = realloc(pVariant->wpz, (*pDestSize + 1) * TSDB_NCHAR_SIZE); + char *tmp = realloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE); assert(tmp != NULL); - pVariant->wpz = (wchar_t *)tmp; + pVariant->ucs4 = (TdUcs4 *)tmp; } else { int32_t output = 0; - bool ret = taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); + bool ret = taosMbsToUcs4(pDst, nLen, (TdUcs4*)*pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); if (!ret) { return -1; } @@ -554,7 +554,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result *result = res; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { errno = 0; - wchar_t *endPtr = NULL; + TdUcs4 *endPtr = NULL; SToken token = {0}; token.n = tGetToken(pVariant->pz, &token.type); @@ -564,7 +564,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } if (token.type == TK_FLOAT) { - double v = wcstod(pVariant->wpz, &endPtr); + double v = wcstod(pVariant->ucs4, &endPtr); if (releaseVariantPtr) { free(pVariant->pz); pVariant->nLen = 0; @@ -583,7 +583,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result setNull((char *)result, type, tDataTypes[type].bytes); return 0; } else { - int64_t val = wcstoll(pVariant->wpz, &endPtr, 10); + int64_t val = wcstoll(pVariant->ucs4, &endPtr, 10); if (releaseVariantPtr) { free(pVariant->pz); pVariant->nLen = 0; @@ -649,7 +649,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) { *pDest = ret; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { int32_t ret = 0; - if ((ret = wcsconvertToBoolImpl(pVariant->wpz, pVariant->nLen)) < 0) { + if ((ret = wcsconvertToBoolImpl(pVariant->ucs4, pVariant->nLen)) < 0) { return ret; } *pDest = ret; @@ -899,7 +899,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc return -1; } } else { - wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen); + tasoUcs4Copy((TdUcs4*)payload, pVariant->ucs4, pVariant->nLen); } } } else { @@ -913,7 +913,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc return -1; } } else { - memcpy(p, pVariant->wpz, pVariant->nLen); + memcpy(p, pVariant->ucs4, pVariant->nLen); newlen = pVariant->nLen; } @@ -979,7 +979,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { pVariant->d = v; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { errno = 0; - double v = wcstod(pVariant->wpz, NULL); + double v = wcstod(pVariant->ucs4, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { free(pVariant->pz); return -1; diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c index 9570bc72a0e620f79a2f50e41ab14408be90e7af..2e5c96ecdda07824fe057400e088c6514d6b736e 100644 --- a/source/dnode/bnode/src/bnode.c +++ b/source/dnode/bnode/src/bnode.c @@ -25,5 +25,3 @@ void bndClose(SBnode *pBnode) { free(pBnode); } int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; } - -void bndDestroy(const char *path) {} diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 64e8980219aa2c0043986898a98e16cc561b8c27..35ea16698378da156fa2caec708944bdef0c1336 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,2 +1,30 @@ -add_subdirectory(daemon) -add_subdirectory(impl) \ No newline at end of file +aux_source_directory(src DNODE_SRC) +aux_source_directory(dnode/src DNODE_SRC) +aux_source_directory(qnode/src DNODE_SRC) +aux_source_directory(bnode/src DNODE_SRC) +aux_source_directory(snode/src DNODE_SRC) +aux_source_directory(vnode/src DNODE_SRC) +aux_source_directory(mnode/src DNODE_SRC) +aux_source_directory(container/src DNODE_SRC) + +add_library(dnode STATIC ${DNODE_SRC}) +target_link_libraries( + dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor +) +target_include_directories( + dnode + PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/snode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mnode/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/container/inc" +) + +add_subdirectory(main) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/bnode/inc/bm.h similarity index 70% rename from source/dnode/mgmt/daemon/inc/dmnInt.h rename to source/dnode/mgmt/bnode/inc/bm.h index 8a571352f021ab53f3405e11114257b7ecbd4fd8..79cf76d11379be81c1b094806b4fadcb97553762 100644 --- a/source/dnode/mgmt/daemon/inc/dmnInt.h +++ b/source/dnode/mgmt/bnode/inc/bm.h @@ -1,4 +1,3 @@ - /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -14,28 +13,19 @@ * along with this program. If not, see . */ -#ifndef _TD_DMN_INT_H_ -#define _TD_DMN_INT_H_ +#ifndef _TD_DND_BNODE_H_ +#define _TD_DND_BNODE_H_ -#include "tconfig.h" -#include "dnode.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tlog.h" -#include "version.h" +#include "dnd.h" #ifdef __cplusplus extern "C" { #endif -SDnodeObjCfg dmnGetObjCfg(); - -void dmnDumpCfg(); -void dmnPrintVersion(); -void dmnGenerateGrant(); +void bmGetMgmtFp(SMgmtWrapper *pWrapper); #ifdef __cplusplus } #endif -#endif /*_TD_DMN_INT_H_*/ +#endif /*_TD_DND_BNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/bnode/inc/bmInt.h b/source/dnode/mgmt/bnode/inc/bmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..ddbf7f4c4e47075b42fce0b259f39456bb3e53f0 --- /dev/null +++ b/source/dnode/mgmt/bnode/inc/bmInt.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_BNODE_INT_H_ +#define _TD_DND_BNODE_INT_H_ + +#include "bm.h" +#include "bnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SBnodeMgmt { + SBnode *pBnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SDnodeWorker writeWorker; +} SBnodeMgmt; + +// bmInt.c +int32_t bmOpen(SMgmtWrapper *pWrapper); +int32_t bmDrop(SMgmtWrapper *pWrapper); + +// bmMsg.c +void bmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// bmWorker.c +int32_t bmStartWorker(SBnodeMgmt *pMgmt); +void bmStopWorker(SBnodeMgmt *pMgmt); +int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_BNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/bnode/src/bmInt.c b/source/dnode/mgmt/bnode/src/bmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..9964b8809082ba430a997d4fee2da809d00495cb --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmInt.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->pWrapper = pMgmt->pWrapper; + pOption->sendReqFp = dndSendReqToDnode; + pOption->sendMnodeReqFp = dndSendReqToMnode; + pOption->sendRspFp = dndSendRsp; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; +} + +static int32_t bmOpenImp(SBnodeMgmt *pMgmt) { + SBnodeOpt option = {0}; + bmInitOption(pMgmt, &option); + + pMgmt->pBnode = bndOpen(pMgmt->path, &option); + if (pMgmt->pBnode == NULL) { + dError("failed to open bnode since %s", terrstr()); + return -1; + } + + if (bmStartWorker(pMgmt) != 0) { + dError("failed to start bnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write bnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void bmCloseImp(SBnodeMgmt *pMgmt) { + if (pMgmt->pBnode != NULL) { + bmStopWorker(pMgmt); + bndClose(pMgmt->pBnode); + pMgmt->pBnode = NULL; + } +} + +int32_t bmDrop(SMgmtWrapper *pWrapper) { + SBnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("bnode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop bnode since %s", terrstr()); + return -1; + } + + bmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("bnode-mgmt is dropped"); + return 0; +} + +static void bmClose(SMgmtWrapper *pWrapper) { + SBnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("bnode-mgmt start to cleanup"); + bmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("bnode-mgmt is cleaned up"); +} + +int32_t bmOpen(SMgmtWrapper *pWrapper) { + dInfo("bnode-mgmt start to init"); + SBnodeMgmt *pMgmt = calloc(1, sizeof(SBnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = bmOpenImp(pMgmt); + if (code != 0) { + dError("failed to init bnode-mgmt since %s", terrstr()); + bmClose(pWrapper); + } else { + dInfo("bnode-mgmt is initialized"); + } + + return code; +} + +void bmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = bmOpen; + mgmtFp.closeFp = bmClose; + mgmtFp.createMsgFp = bmProcessCreateReq; + mgmtFp.dropMsgFp = bmProcessDropReq; + mgmtFp.requiredFp = bmRequire; + + bmInitMsgHandles(pWrapper); + pWrapper->name = "bnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/bnode/src/bmMsg.c b/source/dnode/mgmt/bnode/src/bmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..c01d260c3f1c1c2fae5045765fd953bc15e1f07a --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmMsg.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateBnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return bmOpen(pWrapper); + } +} + +int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropBnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop bnode since %s", terrstr()); + return -1; + } else { + return bmDrop(pWrapper); + } +} + +void bmInitMsgHandles(SMgmtWrapper *pWrapper) {} diff --git a/source/dnode/mgmt/bnode/src/bmWorker.c b/source/dnode/mgmt/bnode/src/bmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..c8ad1378423d871828a92c396e80b26d63a3b76e --- /dev/null +++ b/source/dnode/mgmt/bnode/src/bmWorker.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bmInt.h" + +static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; + dndSendRsp(pWrapper, &rpcRsp); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t numOfMsgs, int32_t code) { + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + bmSendErrorRsp(pWrapper, pMsg, code); + } +} + +static void bmProcessQueue(SBnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) { + SMgmtWrapper *pWrapper = pMgmt->pWrapper; + + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); + if (pArray == NULL) { + bmSendErrorRsps(pWrapper, qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); + return; + } + + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + dTrace("msg:%p, will be processed in bnode queue", pMsg); + if (taosArrayPush(pArray, &pMsg) == NULL) { + bmSendErrorRsp(pWrapper, pMsg, TSDB_CODE_OUT_OF_MEMORY); + } + } + + bndProcessWMsgs(pMgmt->pBnode, pArray); + + for (size_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } + taosArrayDestroy(pArray); +} + +int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->writeWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t bmStartWorker(SBnodeMgmt *pMgmt) { + if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, bmProcessQueue) != 0) { + dError("failed to start bnode write worker since %s", terrstr()); + return -1; + } + + return 0; +} + +void bmStopWorker(SBnodeMgmt *pMgmt) { dndCleanupWorker(&pMgmt->writeWorker); } diff --git a/source/dnode/mgmt/container/inc/dnd.h b/source/dnode/mgmt/container/inc/dnd.h new file mode 100644 index 0000000000000000000000000000000000000000..29c05d6d01e05b2ce5d4d5164ed13e05ba39e773 --- /dev/null +++ b/source/dnode/mgmt/container/inc/dnd.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_H_ +#define _TD_DND_H_ + +#include "os.h" + +#include "cJSON.h" +#include "monitor.h" +#include "tcache.h" +#include "tcrc32c.h" +#include "tdatablock.h" +#include "tglobal.h" +#include "thash.h" +#include "tlockfree.h" +#include "tlog.h" +#include "tmsg.h" +#include "tprocess.h" +#include "tqueue.h" +#include "trpc.h" +#include "tthread.h" +#include "ttime.h" +#include "tworker.h" + +#include "dnode.h" +#include "tfs.h" +#include "wal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} + +typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; +typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; +typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus; +typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; +typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType; + +typedef struct SMgmtFp SMgmtFp; +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SMsgHandle SMsgHandle; +typedef struct SDnodeMgmt SDnodeMgmt; +typedef struct SVnodesMgmt SVnodesMgmt; +typedef struct SMnodeMgmt SMnodeMgmt; +typedef struct SQnodeMgmt SQnodeMgmt; +typedef struct SSnodeMgmt SSnodeMgmt; +typedef struct SBnodeMgmt SBnodeMgmt; + +typedef int32_t (*NodeMsgFp)(void *pMgmt, SNodeMsg *pMsg); +typedef int32_t (*OpenNodeFp)(SMgmtWrapper *pWrapper); +typedef void (*CloseNodeFp)(SMgmtWrapper *pWrapper); +typedef int32_t (*StartNodeFp)(SMgmtWrapper *pWrapper); +typedef int32_t (*CreateNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); + +typedef struct { + EWorkerType type; + const char *name; + int32_t minNum; + int32_t maxNum; + void *queueFp; + void *param; + STaosQueue *queue; + union { + SQWorkerPool pool; + SWWorkerPool mpool; + }; +} SDnodeWorker; + +typedef struct SMsgHandle { + NodeMsgFp msgFp; + SMgmtWrapper *pWrapper; +} SMsgHandle; + +typedef struct SMgmtFp { + OpenNodeFp openFp; + CloseNodeFp closeFp; + StartNodeFp startFp; + CreateNodeFp createMsgFp; + DropNodeFp dropMsgFp; + RequireNodeFp requiredFp; +} SMgmtFp; + +typedef struct SMgmtWrapper { + const char *name; + char *path; + int32_t refCount; + SRWLatch latch; + bool deployed; + bool required; + EProcType procType; + SProcObj *pProc; + void *pMgmt; + SDnode *pDnode; + NodeMsgFp msgFps[TDMT_MAX]; + SMgmtFp fp; +} SMgmtWrapper; + +typedef struct { + void *serverRpc; + void *clientRpc; + SMsgHandle msgHandles[TDMT_MAX]; +} STransMgmt; + +typedef struct SDnode { + int64_t clusterId; + int32_t dnodeId; + int32_t numOfSupportVnodes; + int64_t rebootTime; + char *localEp; + char *localFqdn; + char *firstEp; + char *secondEp; + char *dataDir; + SDiskCfg *pDisks; + int32_t numOfDisks; + uint16_t serverPort; + bool dropped; + EDndStatus status; + EDndEvent event; + EProcType procType; + SStartupReq startup; + TdFilePtr pLockFile; + STransMgmt trans; + SMgmtWrapper wrappers[NODE_MAX]; +} SDnode; + +EDndStatus dndGetStatus(SDnode *pDnode); +void dndSetStatus(SDnode *pDnode, EDndStatus stat); +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp); +void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); +void dndSendMonitorReport(SDnode *pDnode); + +int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg); +void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); + +int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, + int32_t maxNum, void *queueFp); +void dndCleanupWorker(SDnodeWorker *pWorker); +int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg); + +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); + +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/container/inc/dndInt.h b/source/dnode/mgmt/container/inc/dndInt.h new file mode 100644 index 0000000000000000000000000000000000000000..9ae70874feff9e32630491271b0e42665f21b01e --- /dev/null +++ b/source/dnode/mgmt/container/inc/dndInt.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_INT_H_ +#define _TD_DND_INT_H_ + +#include "dnd.h" + +#include "bm.h" +#include "dm.h" +#include "mm.h" +#include "qm.h" +#include "sm.h" +#include "vm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// dndInt.c +int32_t dndInit(); +void dndCleanup(); +const char *dndStatStr(EDndStatus stat); +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); +TdFilePtr dndCheckRunning(char *dataDir); +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); + +// dndMsg.c +void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); + +// dndExec.c +int32_t dndOpenNode(SMgmtWrapper *pWrapper); +void dndCloseNode(SMgmtWrapper *pWrapper); +int32_t dndRun(SDnode *pDnode); + +// dndObj.c +SDnode *dndCreate(const SDnodeOpt *pOption); +void dndClose(SDnode *pDnode); +void dndHandleEvent(SDnode *pDnode, EDndEvent event); + +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); +void dndReleaseWrapper(SMgmtWrapper *pWrapper); + +// dndTransport.c +int32_t dndInitServer(SDnode *pDnode); +void dndCleanupServer(SDnode *pDnode); +int32_t dndInitClient(SDnode *pDnode); +void dndCleanupClient(SDnode *pDnode); +int32_t dndInitMsgHandle(SDnode *pDnode); +void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c new file mode 100644 index 0000000000000000000000000000000000000000..d5c882398ede007bf6b047f2c4c7454b99c5ef73 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static void dndResetLog(SMgmtWrapper *pMgmt) { + char logname[24] = {0}; + snprintf(logname, sizeof(logname), "%slog", pMgmt->name); + + dInfo("node:%s, reset log to %s", pMgmt->name, logname); + taosCloseLog(); + taosInitLog(logname, 1); +} + +static bool dndRequireNode(SMgmtWrapper *pWrapper) { + bool required = false; + int32_t code =(*pWrapper->fp.requiredFp)(pWrapper, &required); + if (!required) { + dDebug("node:%s, no need to start", pWrapper->name); + } else { + dDebug("node:%s, need to start", pWrapper->name); + } + return required; +} + +int32_t dndOpenNode(SMgmtWrapper *pWrapper) { + int32_t code = (*pWrapper->fp.openFp)(pWrapper); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + return -1; + } else { + dDebug("node:%s, has been opened", pWrapper->name); + } + + pWrapper->deployed = true; + return 0; +} + +void dndCloseNode(SMgmtWrapper *pWrapper) { + taosWLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + (*pWrapper->fp.closeFp)(pWrapper); + pWrapper->deployed = false; + } + if (pWrapper->pProc) { + taosProcCleanup(pWrapper->pProc); + pWrapper->pProc = NULL; + } + taosWUnLockLatch(&pWrapper->latch); +} + +static int32_t dndRunInSingleProcess(SDnode *pDnode) { + dInfo("dnode run in single process mode"); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + pWrapper->required = dndRequireNode(pWrapper); + if (!pWrapper->required) continue; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + dInfo("node:%s, will start in single process", pWrapper->name); + pWrapper->procType = PROC_SINGLE; + if (dndOpenNode(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + } + + dndSetStatus(pDnode, DND_STAT_RUNNING); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + if (!pWrapper->required) continue; + if (pWrapper->fp.startFp == NULL) continue; + if ((*pWrapper->fp.startFp)(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + } + + return 0; +} + +static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) { + dndCleanupServer(pDnode); + for (ENodeType n = 0; n < NODE_MAX; ++n) { + if (except == n) continue; + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + dndCloseNode(pWrapper); + } +} + +static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) { + dTrace("msg:%p, get from child queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + pRpc->pCont = pCont; + + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + int32_t code = (*msgFp)(pWrapper, pMsg); + + if (code != 0) { + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pCont); + } +} + +static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { + dTrace("msg:%p, get from parent queue", pRsp); + pRsp->pCont = pCont; + dndSendRpcRsp(pWrapper, pRsp); + free(pRsp); +} + +static int32_t dndRunInMultiProcess(SDnode *pDnode) { + dInfo("dnode run in multi process mode"); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + pWrapper->required = dndRequireNode(pWrapper); + if (!pWrapper->required) continue; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + if (n == DNODE) { + dInfo("node:%s, will start in parent process", pWrapper->name); + pWrapper->procType = PROC_SINGLE; + if (dndOpenNode(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + continue; + } + + SProcCfg cfg = {.childQueueSize = 1024 * 1024 * 2, // size will be a configuration item + .childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentQueueSize = 1024 * 1024 * 2, // size will be a configuration item + .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, + .parentdMallocHeadFp = (ProcMallocFp)malloc, + .parentFreeHeadFp = (ProcFreeFp)free, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .testFlag = 0, + .pParent = pWrapper, + .name = pWrapper->name}; + SProcObj *pProc = taosProcInit(&cfg); + if (pProc == NULL) { + dError("node:%s, failed to fork since %s", pWrapper->name, terrstr()); + return -1; + } + + pWrapper->pProc = pProc; + + if (taosProcIsChild(pProc)) { + dInfo("node:%s, will start in child process", pWrapper->name); + pWrapper->procType = PROC_CHILD; + dndResetLog(pWrapper); + + dInfo("node:%s, clean up resources inherited from parent", pWrapper->name); + dndClearNodesExecpt(pDnode, n); + + dInfo("node:%s, will be initialized in child process", pWrapper->name); + dndOpenNode(pWrapper); + } else { + dInfo("node:%s, will not start in parent process", pWrapper->name); + pWrapper->procType = PROC_PARENT; + } + + if (taosProcRun(pProc) != 0) { + dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); + return -1; + } + } + +#if 0 + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper->procType == PROC_PARENT && dmStart(pWrapper->pMgmt) != 0) { + dndReleaseWrapper(pWrapper); + dError("failed to start dnode worker since %s", terrstr()); + return -1; + } + + dndReleaseWrapper(pWrapper); +#endif + return 0; +} + +int32_t dndRun(SDnode *pDnode) { + if (tsMultiProcess == 0) { + if (dndRunInSingleProcess(pDnode) != 0) { + dError("failed to run dnode in single process mode since %s", terrstr()); + return -1; + } + } else { + if (dndRunInMultiProcess(pDnode) != 0) { + dError("failed to run dnode in multi process mode since %s", terrstr()); + return -1; + } + } + + dndReportStartup(pDnode, "TDengine", "initialized successfully"); + + while (1) { + if (pDnode->event == DND_EVENT_STOP) { + dInfo("dnode is about to stop"); + break; + } + taosMsleep(100); + } + + return 0; +} diff --git a/source/dnode/mgmt/container/src/dndFile.c b/source/dnode/mgmt/container/src/dndFile.c new file mode 100644 index 0000000000000000000000000000000000000000..a5f8fb71690c86b355ea2d5d131ce20160d972b8 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndFile.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 1024; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto _OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto _OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto _OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", file); + goto _OVER; + } + *pDeployed = deployed->valueint != 0; + + code = 0; + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + +_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 1024; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + free(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d", realfile, deployed); + return 0; +} diff --git a/source/dnode/mgmt/container/src/dndInt.c b/source/dnode/mgmt/container/src/dndInt.c new file mode 100644 index 0000000000000000000000000000000000000000..4decc6ba87323533f651311b41aa0c0e58a95555 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndInt.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static int8_t once = DND_ENV_INIT; + +int32_t dndInit() { + dDebug("start to init dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { + terrno = TSDB_CODE_REPEAT_INIT; + dError("failed to init dnode env since %s", terrstr()); + return -1; + } + + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + + if (rpcInit() != 0) { + dError("failed to init rpc since %s", terrstr()); + dndCleanup(); + return -1; + } + + SMonCfg monCfg = {0}; + monCfg.maxLogs = tsMonitorMaxLogs; + monCfg.port = tsMonitorPort; + monCfg.server = tsMonitorFqdn; + monCfg.comp = tsMonitorComp; + if (monInit(&monCfg) != 0) { + dError("failed to init monitor since %s", terrstr()); + dndCleanup(); + return -1; + } + + dInfo("dnode env is initialized"); + return 0; +} + +void dndCleanup() { + dDebug("start to cleanup dnode env"); + if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + dError("dnode env is already cleaned up"); + return; + } + + monCleanup(); + rpcCleanup(); + walCleanUp(); + taosStopCacheRefreshWorker(); + dInfo("dnode env is cleaned up"); +} + +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp) { + pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; +} + +EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } + +void dndSetStatus(SDnode *pDnode, EDndStatus status) { + if (pDnode->status != status) { + dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); + pDnode->status = status; + } +} + +const char *dndStatStr(EDndStatus status) { + switch (status) { + case DND_STAT_INIT: + return "init"; + case DND_STAT_RUNNING: + return "running"; + case DND_STAT_STOPPED: + return "stopped"; + default: + return "unknown"; + } +} + +void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} + +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +TdFilePtr dndCheckRunning(char *dataDir) { + char filepath[PATH_MAX] = {0}; + snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s, quit", filepath, terrstr()); + return NULL; + } + + int32_t ret = taosLockFile(pFile); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s, quit", filepath, terrstr()); + taosCloseFile(&pFile); + return NULL; + } + + dDebug("file:%s is locked", filepath); + return pFile; +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + + SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; + rpcSendResponse(&rpcRsp); +} diff --git a/source/dnode/mgmt/container/src/dndMonitor.c b/source/dnode/mgmt/container/src/dndMonitor.c new file mode 100644 index 0000000000000000000000000000000000000000..ef3db387de8deff0c26f0dac67f93c4740013089 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndMonitor.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; + + return vmMonitorTfsInfo(dndAcquireWrapper(pDnode, VNODES), pInfo); +} + +static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { + pInfo->protocol = 1; + pInfo->dnode_id = pDnode->dnodeId; + pInfo->cluster_id = pDnode->clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + +static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { + pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f); + taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); + taosGetCpuCores(&pInfo->cpu_cores); + taosGetProcMemory(&pInfo->mem_engine); + taosGetSysMemory(&pInfo->mem_system); + pInfo->mem_total = tsTotalMemoryKB; + pInfo->disk_engine = 0; + pInfo->disk_used = tsDataSpace.size.used; + pInfo->disk_total = tsDataSpace.size.total; + taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); + taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); + + vmMonitorVnodeReqs(dndAcquireWrapper(pDnode, VNODES), pInfo); + pInfo->has_mnode = (dndAcquireWrapper(pDnode, MNODE)->required); +} + +void dndSendMonitorReport(SDnode *pDnode) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + SMonBasicInfo basicInfo = {0}; + dndGetMonitorBasicInfo(pDnode, &basicInfo); + monSetBasicInfo(pMonitor, &basicInfo); + + SMonClusterInfo clusterInfo = {0}; + SMonVgroupInfo vgroupInfo = {0}; + SMonGrantInfo grantInfo = {0}; + if (mmMonitorMnodeInfo(dndAcquireWrapper(pDnode, MNODE), &clusterInfo, &vgroupInfo, &grantInfo) == 0) { + monSetClusterInfo(pMonitor, &clusterInfo); + monSetVgroupInfo(pMonitor, &vgroupInfo); + monSetGrantInfo(pMonitor, &grantInfo); + } + + SMonDnodeInfo dnodeInfo = {0}; + dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); + monSetDnodeInfo(pMonitor, &dnodeInfo); + + SMonDiskInfo diskInfo = {0}; + if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { + monSetDiskInfo(pMonitor, &diskInfo); + } + + taosArrayDestroy(clusterInfo.dnodes); + taosArrayDestroy(clusterInfo.mnodes); + taosArrayDestroy(vgroupInfo.vgroups); + taosArrayDestroy(diskInfo.datadirs); + + monSendReport(pMonitor); + monCleanupMonitorInfo(pMonitor); +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..21e9cc71a54ee790713dbc761440b0c435dd16fc --- /dev/null +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); + } + dndReleaseWrapper(pWrapper); +} + +static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + if (msgFp == NULL) { + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + } + + return msgFp; +} + +static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { + SRpcConnInfo connInfo = {0}; + if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle); + return -1; + } + + memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); + memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); + + return 0; +} + +void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + + int32_t code = -1; + SNodeMsg *pMsg = NULL; + NodeMsgFp msgFp = NULL; + + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; + if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; + + dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); + if (pWrapper->procType == PROC_SINGLE) { + code = (*msgFp)(pWrapper->pMgmt, pMsg); + } else if (pWrapper->procType == PROC_PARENT) { + code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen); + } else { + } + +_OVER: + if (code == 0) { + if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + } else { + dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + dndReleaseWrapper(pWrapper); +} + +static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper != NULL) { + dndReleaseWrapper(pWrapper); + terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + dError("failed to create node since %s", terrstr()); + return -1; + } + + pWrapper = &pDnode->wrappers[ntype]; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been opened", pWrapper->name); + pWrapper->deployed = true; + } + + return code; +} + +static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + dError("failed to drop node since %s", terrstr()); + return -1; + } + + taosWLockLatch(&pWrapper->latch); + pWrapper->deployed = false; + + int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); + if (code != 0) { + pWrapper->deployed = true; + dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); + } else { + pWrapper->deployed = false; + dDebug("node:%s, has been dropped", pWrapper->name); + } + + taosWUnLockLatch(&pWrapper->latch); + dndReleaseWrapper(pWrapper); + return code; +} + +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { + switch (pMsg->rpcMsg.msgType) { + case TDMT_DND_CREATE_MNODE: + return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_DROP_MNODE: + return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_CREATE_QNODE: + return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_DROP_QNODE: + return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_CREATE_SNODE: + return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_DROP_SNODE: + return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_CREATE_BNODE: + return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); + case TDMT_DND_DROP_BNODE: + return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + return -1; + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndObj.c b/source/dnode/mgmt/container/src/dndObj.c new file mode 100644 index 0000000000000000000000000000000000000000..75a06d585920735b336202c2db7d52e5fd11b0fd --- /dev/null +++ b/source/dnode/mgmt/container/src/dndObj.c @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) { + pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes; + pDnode->serverPort = pOption->serverPort; + pDnode->dataDir = strdup(pOption->dataDir); + pDnode->localEp = strdup(pOption->localEp); + pDnode->localFqdn = strdup(pOption->localFqdn); + pDnode->firstEp = strdup(pOption->firstEp); + pDnode->secondEp = strdup(pOption->secondEp); + pDnode->pDisks = pOption->pDisks; + pDnode->numOfDisks = pOption->numOfDisks; + pDnode->rebootTime = taosGetTimestampMs(); + + if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL || + pDnode->secondEp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndClearMemory(SDnode *pDnode) { + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pMgmt = &pDnode->wrappers[n]; + tfree(pMgmt->path); + } + if (pDnode->pLockFile != NULL) { + taosUnLockFile(pDnode->pLockFile); + taosCloseFile(&pDnode->pLockFile); + pDnode->pLockFile = NULL; + } + tfree(pDnode->localEp); + tfree(pDnode->localFqdn); + tfree(pDnode->firstEp); + tfree(pDnode->secondEp); + tfree(pDnode->dataDir); + free(pDnode); + dDebug("dnode object memory is cleared, data:%p", pDnode); +} + +SDnode *dndCreate(const SDnodeOpt *pOption) { + dInfo("start to create dnode object"); + int32_t code = -1; + char path[PATH_MAX]; + SDnode *pDnode = NULL; + + pDnode = calloc(1, sizeof(SDnode)); + if (pDnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (dndInitMemory(pDnode, pOption) != 0) { + goto _OVER; + } + + dndSetStatus(pDnode, DND_STAT_INIT); + pDnode->pLockFile = dndCheckRunning(pDnode->dataDir); + if (pDnode->pLockFile == NULL) { + goto _OVER; + } + + if (dndInitServer(pDnode) != 0) { + dError("failed to init trans server since %s", terrstr()); + goto _OVER; + } + + if (dndInitClient(pDnode) != 0) { + dError("failed to init trans client since %s", terrstr()); + goto _OVER; + } + + dmGetMgmtFp(&pDnode->wrappers[DNODE]); + mmGetMgmtFp(&pDnode->wrappers[MNODE]); + vmGetMgmtFp(&pDnode->wrappers[VNODES]); + qmGetMgmtFp(&pDnode->wrappers[QNODE]); + smGetMgmtFp(&pDnode->wrappers[SNODE]); + bmGetMgmtFp(&pDnode->wrappers[BNODE]); + + if (dndInitMsgHandle(pDnode) != 0) { + goto _OVER; + } + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name); + pWrapper->path = strdup(path); + pWrapper->pDnode = pDnode; + if (pWrapper->path == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + pWrapper->procType = PROC_SINGLE; + taosInitRWLatch(&pWrapper->latch); + } + + code = 0; + +_OVER: + if (code != 0 && pDnode) { + dndClearMemory(pDnode); + dError("failed to create dnode object since %s", terrstr()); + } else { + dInfo("dnode object is created, data:%p", pDnode); + } + + return pDnode; +} + +void dndClose(SDnode *pDnode) { + if (pDnode == NULL) return; + + if (dndGetStatus(pDnode) == DND_STAT_STOPPED) { + dError("dnode is shutting down, data:%p", pDnode); + return; + } + + dInfo("start to close dnode, data:%p", pDnode); + dndSetStatus(pDnode, DND_STAT_STOPPED); + + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + dndCloseNode(pWrapper); + } + + dndClearMemory(pDnode); + dInfo("dnode object is closed, data:%p", pDnode); +} + +void dndHandleEvent(SDnode *pDnode, EDndEvent event) { + dInfo("dnode object receive event %d, data:%p", event, pDnode); + pDnode->event = event; +} + +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + SMgmtWrapper *pRetWrapper = pWrapper; + + taosRLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); + dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount); + } else { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + pRetWrapper = NULL; + } + taosRUnLockLatch(&pWrapper->latch); + + return pRetWrapper; +} + +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) { + int32_t code = 0; + + taosRLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); + dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount); + } else { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + code = -1; + } + taosRUnLockLatch(&pWrapper->latch); + + return code; +} + +void dndReleaseWrapper(SMgmtWrapper *pWrapper) { + if (pWrapper == NULL) return; + + taosRLockLatch(&pWrapper->latch); + int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); + taosRUnLockLatch(&pWrapper->latch); + dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount); +} \ No newline at end of file diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c new file mode 100644 index 0000000000000000000000000000000000000000..2593a762e8236cd6cda4cbcc6ada40df42f5c2f2 --- /dev/null +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndInt.h" + +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" +#define INTERNAL_SECRET "_pwd" + +static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { + SDnode *pDnode = parent; + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pRsp->msgType; + + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + // if (pRsp == NULL || pRsp->pCont == NULL) return; + dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); + rpcFreeCont(pRsp->pCont); + return; + } + + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + if (pHandle->msgFp != NULL) { + dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), + pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); + dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); + } else { + dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); + rpcFreeCont(pRsp->pCont); + } +} + +int32_t dndInitClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = dndProcessResponse; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = INTERNAL_USER; + rpcInit.ckey = INTERNAL_CKEY; + rpcInit.spi = 1; + rpcInit.parent = pDnode; + + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + rpcInit.secret = pass; + + pMgmt->clientRpc = rpcOpen(&rpcInit); + if (pMgmt->clientRpc == NULL) { + dError("failed to init dnode rpc client"); + return -1; + } + + dDebug("dnode rpc client is initialized"); + return 0; +} + +void dndCleanupClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->clientRpc) { + rpcClose(pMgmt->clientRpc); + pMgmt->clientRpc = NULL; + dDebug("dnode rpc client is closed"); + } +} + +static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { + SDnode *pDnode = param; + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pReq->msgType; + + if (msgType == TDMT_DND_NETWORK_TEST) { + dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle); + dndProcessStartupReq(pDnode, pReq); + return; + } + + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pReq->pCont); + return; + } + + if (pReq->pCont == NULL) { + dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + return; + } + + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + if (pHandle->msgFp != NULL) { + dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, + pReq->handle, pReq->ahandle); + dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); + } else { + dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pReq->pCont); + } +} + +static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { + STransMgmt *pMgmt = &pDnode->trans; + + SEpSet epSet = {0}; + dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet); + rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); +} + +static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + int32_t code = 0; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + + if (strcmp(user, INTERNAL_USER) == 0) { + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { + taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); + } else { + code = -1; + } + + if (code == 0) { + memcpy(secret, pass, TSDB_PASSWORD_LEN); + *spi = 1; + *encrypt = 0; + *ckey = 0; + } + + return code; +} + +static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SDnode *pDnode = parent; + + if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) { + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + + if (mmGetUserAuth(dndAcquireWrapper(pDnode, MNODE), user, spi, encrypt, secret, ckey) == 0) { + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + + if (terrno != TSDB_CODE_APP_NOT_READY) { + dTrace("failed to get user auth from mnode since %s", terrstr()); + return -1; + } + + SAuthReq authReq = {0}; + tstrncpy(authReq.user, user, TSDB_USER_LEN); + int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSAuthReq(pReq, contLen, &authReq); + + SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; + SRpcMsg rpcRsp = {0}; + dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); + dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + terrno = rpcRsp.code; + dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); + } else { + SAuthRsp authRsp = {0}; + tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); + memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); + memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); + *spi = authRsp.spi; + *encrypt = authRsp.encrypt; + dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, + authRsp.encrypt); + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; +} + +int32_t dndInitServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); + if (numOfThreads < 1) { + numOfThreads = 1; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = pDnode->serverPort; + rpcInit.label = "DND"; + rpcInit.numOfThreads = numOfThreads; + rpcInit.cfp = dndProcessRequest; + rpcInit.sessions = tsMaxShellConns; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.afp = dndRetrieveUserAuthInfo; + rpcInit.parent = pDnode; + + pMgmt->serverRpc = rpcOpen(&rpcInit); + if (pMgmt->serverRpc == NULL) { + dError("failed to init dnode rpc server"); + return -1; + } + + dDebug("dnode rpc server is initialized"); + return 0; +} + +void dndCleanupServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + if (pMgmt->serverRpc) { + rpcClose(pMgmt->serverRpc); + pMgmt->serverRpc = NULL; + dDebug("dnode rpc server is closed"); + } +} + +int32_t dndInitMsgHandle(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->trans; + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + + for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { + NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; + if (msgFp == NULL) continue; + + SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; + if (pHandle->msgFp != NULL) { + dError("msg:%s has multiple process nodes, prev node:%s, curr node:%s", tMsgInfo[msgIndex], + pHandle->pWrapper->name, pWrapper->name); + return -1; + } else { + dTrace("msg:%s will be processed by %s", tMsgInfo[msgIndex], pWrapper->name); + pHandle->msgFp = msgFp; + pHandle->pWrapper = pWrapper; + } + } + } + + return 0; +} + +static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) { + if (pMgmt->clientRpc == NULL) { + terrno = TSDB_CODE_DND_OFFLINE; + return -1; + } + + rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); + return 0; +} + +int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq) { + if (pWrapper->procType == PROC_CHILD) { + } else { + SDnode *pDnode = pWrapper->pDnode; + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + terrno = TSDB_CODE_DND_OFFLINE; + dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); + return -1; + } + return dndSendRpcReq(&pDnode->trans, pEpSet, pReq); + } +} + +int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { + if (pWrapper->procType == PROC_CHILD) { + } else { + SDnode *pDnode = pWrapper->pDnode; + STransMgmt *pTrans = &pDnode->trans; + SEpSet epSet = {0}; + dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet); + return dndSendRpcReq(pTrans, &epSet, pReq); + } +} + +void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { + if (pRsp->code == TSDB_CODE_APP_NOT_READY) { + SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); + dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); + } else { + rpcSendResponse(pRsp); + } +} + +void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { + if (pWrapper->procType == PROC_CHILD) { + int32_t code = -1; + do { + code = taosProcPutToParentQueue(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen); + if (code != 0) { + taosMsleep(10); + } + } while (code != 0); + } else { + dndSendRpcRsp(pWrapper, pRsp); + } +} diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/container/src/dndWorker.c similarity index 74% rename from source/dnode/mgmt/impl/src/dndWorker.c rename to source/dnode/mgmt/container/src/dndWorker.c index 38f8737b2bbbc0796a98688498eeab7d44cdef76..5d99ec5f0d99fa805b3631d237e9aef212e06398 100644 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ b/source/dnode/mgmt/container/src/dndWorker.c @@ -14,11 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "dndWorker.h" +#include "dndInt.h" -int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, +int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, int32_t maxNum, void *queueFp) { - if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) { + if (pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) { terrno = TSDB_CODE_INVALID_PARA; return -1; } @@ -28,7 +28,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c pWorker->minNum = minNum; pWorker->maxNum = maxNum; pWorker->queueFp = queueFp; - pWorker->pDnode = pDnode; + pWorker->param = param; if (pWorker->type == DND_WORKER_SINGLE) { SQWorkerPool *pPool = &pWorker->pool; @@ -39,7 +39,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pWorker->queue = tQWorkerAllocQueue(pPool, pDnode, (FItem)queueFp); + pWorker->queue = tQWorkerAllocQueue(pPool, param, (FItem)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -52,7 +52,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pWorker->queue = tWWorkerAllocQueue(pPool, pDnode, (FItems)queueFp); + pWorker->queue = tWWorkerAllocQueue(pPool, param, (FItems)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -65,6 +65,8 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c } void dndCleanupWorker(SDnodeWorker *pWorker) { + if (pWorker->queue == NULL) return; + while (!taosQueueEmpty(pWorker->queue)) { taosMsleep(10); } @@ -79,31 +81,13 @@ void dndCleanupWorker(SDnodeWorker *pWorker) { } } -int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) { +int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg) { if (pWorker == NULL || pWorker->queue == NULL) { terrno = TSDB_CODE_INVALID_PARA; return -1; } - void *pMsg = NULL; - if (contLen != 0) { - pMsg = taosAllocateQitem(contLen); - if (pMsg != NULL) { - memcpy(pMsg, pCont, contLen); - } - } else { - pMsg = pCont; - } - - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - if (taosWriteQitem(pWorker->queue, pMsg) != 0) { - if (contLen != 0) { - taosFreeQitem(pMsg); - } terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c deleted file mode 100644 index afb9ffbdd0b43cf22e8fdcacf4eabb8156faf2d3..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmnInt.h" -#include "tconfig.h" - -SDnodeObjCfg dmnGetObjCfg() { - SConfig *pCfg = taosGetCfg(); - SDnodeObjCfg objCfg = {0}; - - objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; - tstrncpy(objCfg.dataDir, tsDataDir, sizeof(objCfg.dataDir)); - tstrncpy(objCfg.firstEp, tsFirst, sizeof(objCfg.firstEp)); - tstrncpy(objCfg.secondEp, tsSecond, sizeof(objCfg.firstEp)); - objCfg.serverPort = tsServerPort; - tstrncpy(objCfg.localFqdn, tsLocalFqdn, sizeof(objCfg.localFqdn)); - snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); - objCfg.pDisks = tsDiskCfg; - objCfg.numOfDisks = tsDiskCfgNum; - return objCfg; -} - -void dmnDumpCfg() { - SConfig *pCfg = taosGetCfg(); - cfgDumpCfg(pCfg, 0, 1); -} \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c deleted file mode 100644 index 7ba272453ca247600024411c106af17a1dc6cd25..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/daemon/src/dmnMain.c +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmnInt.h" - -static struct { - bool stop; - bool dumpConfig; - bool generateGrant; - bool printAuth; - bool printVersion; - char envFile[PATH_MAX]; - char apolloUrl[PATH_MAX]; -} dmn = {0}; - -static void dmnSigintHandle(int signum, void *info, void *ctx) { - uInfo("singal:%d is received", signum); - dmn.stop = true; -} - -static void dmnSetSignalHandle() { - taosSetSignal(SIGTERM, dmnSigintHandle); - taosSetSignal(SIGHUP, dmnSigintHandle); - taosSetSignal(SIGINT, dmnSigintHandle); - taosSetSignal(SIGABRT, dmnSigintHandle); - taosSetSignal(SIGBREAK, dmnSigintHandle); -} - -static void dmnWaitSignal() { - dmnSetSignalHandle(); - while (!dmn.stop) { - taosMsleep(100); - } -} - -static int32_t dmnParseOption(int32_t argc, char const *argv[]) { - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("config file path overflow"); - return -1; - } - tstrncpy(configDir, argv[i], PATH_MAX); - } else { - printf("'-c' requires a parameter, default is %s\n", configDir); - return -1; - } - } else if (strcmp(argv[i], "-C") == 0) { - dmn.dumpConfig = true; - } else if (strcmp(argv[i], "-k") == 0) { - dmn.generateGrant = true; - } else if (strcmp(argv[i], "-V") == 0) { - dmn.printVersion = true; - } else { - } - } - - return 0; -} - -int32_t dmnRunDnode() { - if (dndInit() != 0) { - uInfo("Failed to start TDengine, please check the log"); - return -1; - } - - SDnodeObjCfg objCfg = dmnGetObjCfg(); - SDnode *pDnode = dndCreate(&objCfg); - if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log"); - return -1; - } - - uInfo("Started TDengine service successfully."); - dmnWaitSignal(); - uInfo("TDengine is shut down!"); - - dndClose(pDnode); - dndCleanup(); - taosCloseLog(); - taosCleanupCfg(); - return 0; -} - -int main(int argc, char const *argv[]) { - if (!taosCheckSystemIsSmallEnd()) { - uError("TDengine does not run on non-small-end machines."); - return -1; - } - - if (dmnParseOption(argc, argv) != 0) { - return -1; - } - - if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; - } - - if (dmn.printVersion) { - dmnPrintVersion(); - return 0; - } - - if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { - uInfo("Failed to start TDengine since read config error"); - return -1; - } - - if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { - uInfo("Failed to start TDengine since read config error"); - return -1; - } - - if (dmn.dumpConfig) { - dmnDumpCfg(); - taosCleanupCfg(); - return 0; - } - - return dmnRunDnode(); -} diff --git a/source/dnode/mgmt/dnode/inc/dm.h b/source/dnode/mgmt/dnode/inc/dm.h new file mode 100644 index 0000000000000000000000000000000000000000..6c18d7969cab87627a0defd4b51e6ebf612b57d8 --- /dev/null +++ b/source/dnode/mgmt/dnode/inc/dm.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_DNODE_H_ +#define _TD_DND_DNODE_H_ + +#include "dnd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SDnodeMgmt SDnodeMgmt; + +void dmGetMgmtFp(SMgmtWrapper *pWrapper); +void dmInitMsgHandles(SMgmtWrapper *pWrapper); + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dnode/inc/dmInt.h b/source/dnode/mgmt/dnode/inc/dmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..46e70727af03548f292cc44ab81771353ae9f52e --- /dev/null +++ b/source/dnode/mgmt/dnode/inc/dmInt.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_DNODE_INT_H_ +#define _TD_DND_DNODE_INT_H_ + +#include "dm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SDnodeMgmt { + int64_t dver; + int64_t updateTime; + int8_t statusSent; + SEpSet mnodeEpSet; + SHashObj *dnodeHash; + SArray *dnodeEps; + TdThread *threadId; + SRWLatch latch; + SDnodeWorker mgmtWorker; + SDnodeWorker statusWorker; + const char *path; + SDnode *pDnode; + SMgmtWrapper *pWrapper; +} SDnodeMgmt; + +// dmFile.c +int32_t dmReadFile(SDnodeMgmt *pMgmt); +int32_t dmWriteFile(SDnodeMgmt *pMgmt); +void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); + +// dmMsg.c +void dmSendStatusReq(SDnodeMgmt *pMgmt); +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); + +// dmWorker.c +int32_t dmStartWorker(SDnodeMgmt *pMgmt); +void dmStopWorker(SDnodeMgmt *pMgmt); +int32_t dmStartThread(SDnodeMgmt *pMgmt); +int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_DNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dnode/src/dmFile.c b/source/dnode/mgmt/dnode/src/dmFile.c new file mode 100644 index 0000000000000000000000000000000000000000..aa2247bcd5939f5268eaa141af2acc6f35474023 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmFile.c @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +static void dmPrintDnodes(SDnodeMgmt *pMgmt); +static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep); +static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps); + +int32_t dmReadFile(SDnodeMgmt *pMgmt) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 256 * 1024; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + SDnode *pDnode = pMgmt->pDnode; + + pMgmt->dnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); + if (pMgmt->dnodeEps == NULL) { + dError("failed to calloc dnodeEp array since %s", strerror(errno)); + goto PRASE_DNODE_OVER; + } + + snprintf(file, sizeof(file), "%s%sdnode.json", pMgmt->path, TD_DIRSEP); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_DNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_DNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_DNODE_OVER; + } + + cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s since dnodeId not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->dnodeId = dnodeId->valueint; + + cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); + if (!clusterId || clusterId->type != cJSON_String) { + dError("failed to read %s since clusterId not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->clusterId = atoll(clusterId->valuestring); + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_DNODE_OVER; + } + pDnode->dropped = dropped->valueint; + + cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); + if (!dnodes || dnodes->type != cJSON_Array) { + dError("failed to read %s since dnodes not found", file); + goto PRASE_DNODE_OVER; + } + + int32_t numOfDnodes = cJSON_GetArraySize(dnodes); + if (numOfDnodes <= 0) { + dError("failed to read %s since numOfDnodes:%d invalid", file, numOfDnodes); + goto PRASE_DNODE_OVER; + } + + for (int32_t i = 0; i < numOfDnodes; ++i) { + cJSON *node = cJSON_GetArrayItem(dnodes, i); + if (node == NULL) break; + + SDnodeEp dnodeEp = {0}; + + cJSON *did = cJSON_GetObjectItem(node, "id"); + if (!did || did->type != cJSON_Number) { + dError("failed to read %s since dnodeId not found", file); + goto PRASE_DNODE_OVER; + } + + dnodeEp.id = dnodeId->valueint; + + cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); + if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { + dError("failed to read %s since dnodeFqdn not found", file); + goto PRASE_DNODE_OVER; + } + tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); + if (!dnodePort || dnodePort->type != cJSON_Number) { + dError("failed to read %s since dnodePort not found", file); + goto PRASE_DNODE_OVER; + } + + dnodeEp.ep.port = dnodePort->valueint; + + cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); + if (!isMnode || isMnode->type != cJSON_Number) { + dError("failed to read %s since isMnode not found", file); + goto PRASE_DNODE_OVER; + } + dnodeEp.isMnode = isMnode->valueint; + + taosArrayPush(pMgmt->dnodeEps, &dnodeEp); + } + + code = 0; + dInfo("succcessed to read file %s", file); + dmPrintDnodes(pMgmt); + +PRASE_DNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + if (dmIsEpChanged(pMgmt, pDnode->dnodeId, pDnode->localEp)) { + dError("localEp %s different with %s and need reconfigured", pDnode->localEp, file); + return -1; + } + + if (taosArrayGetSize(pMgmt->dnodeEps) == 0) { + SDnodeEp dnodeEp = {0}; + dnodeEp.isMnode = 1; + taosGetFqdnPortFromEp(pDnode->firstEp, &dnodeEp.ep); + taosArrayPush(pMgmt->dnodeEps, &dnodeEp); + } + + dmResetDnodes(pMgmt, pMgmt->dnodeEps); + + terrno = code; + return code; +} + +int32_t dmWriteFile(SDnodeMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%sdnode.json.bak", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + dError("failed to write %s since %s", file, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 256 * 1024; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pDnode->dnodeId); + len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pDnode->clusterId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pDnode->dropped); + len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); + + int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + for (int32_t i = 0; i < numOfEps; ++i) { + SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->dnodeEps, i); + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); + len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); + if (i < numOfEps - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + free(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + pMgmt->updateTime = taosGetTimestampMs(); + dDebug("successed to write %s", file); + return 0; +} + +void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *dnodeEps) { + int32_t numOfEps = taosArrayGetSize(dnodeEps); + if (numOfEps <= 0) return; + + taosWLockLatch(&pMgmt->latch); + + int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + if (numOfEps != numOfEpsOld) { + dmResetDnodes(pMgmt, dnodeEps); + dmWriteFile(pMgmt); + } else { + int32_t size = numOfEps * sizeof(SDnodeEp); + if (memcmp(pMgmt->dnodeEps->pData, dnodeEps->pData, size) != 0) { + dmResetDnodes(pMgmt, dnodeEps); + dmWriteFile(pMgmt); + } + } + + taosWUnLockLatch(&pMgmt->latch); +} + +static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps) { + if (pMgmt->dnodeEps != dnodeEps) { + SArray *tmp = pMgmt->dnodeEps; + pMgmt->dnodeEps = taosArrayDup(dnodeEps); + taosArrayDestroy(tmp); + } + + pMgmt->mnodeEpSet.inUse = 0; + pMgmt->mnodeEpSet.numOfEps = 0; + + int32_t mIndex = 0; + int32_t numOfEps = (int32_t)taosArrayGetSize(dnodeEps); + + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); + if (!pDnodeEp->isMnode) continue; + if (mIndex >= TSDB_MAX_REPLICA) continue; + pMgmt->mnodeEpSet.numOfEps++; + + pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep; + mIndex++; + } + + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); + taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); + } + + dmPrintDnodes(pMgmt); +} + +static void dmPrintDnodes(SDnodeMgmt *pMgmt) { + int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps); + dDebug("print dnode ep list, num:%d", numOfEps); + for (int32_t i = 0; i < numOfEps; i++) { + SDnodeEp *pEp = taosArrayGet(pMgmt->dnodeEps, i); + dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); + } +} + +static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep) { + bool changed = false; + taosRLockLatch(&pMgmt->latch); + + SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); + if (pDnodeEp != NULL) { + char epstr[TSDB_EP_LEN + 1]; + snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); + changed = strcmp(ep, epstr) != 0; + } + + taosRUnLockLatch(&pMgmt->latch); + return changed; +} diff --git a/source/dnode/mgmt/dnode/src/dmInt.c b/source/dnode/mgmt/dnode/src/dmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..1746cbe6e106a26eb7f8644f399ed8ce0b1b2b65 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmInt.c @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) { + taosRLockLatch(&pMgmt->latch); + *pEpSet = pMgmt->mnodeEpSet; + taosRUnLockLatch(&pMgmt->latch); +} + +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) { + dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); + + taosWLockLatch(&pMgmt->latch); + pMgmt->mnodeEpSet = *pEpSet; + for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { + dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } + + taosWUnLockLatch(&pMgmt->latch); +} + +void dmGetDnodeEp(SMgmtWrapper *pWrapper, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + taosRLockLatch(&pMgmt->latch); + + SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); + if (pDnodeEp != NULL) { + if (pPort != NULL) { + *pPort = pDnodeEp->ep.port; + } + if (pFqdn != NULL) { + tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); + } + if (pEp != NULL) { + snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); + } + } + + taosRUnLockLatch(&pMgmt->latch); +} + +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) { + SDnode *pDnode = pMgmt->pDnode; + + SEpSet epSet = {0}; + dmGetMnodeEpSet(pMgmt, &epSet); + + dDebug("RPC %p, req is redirected, num:%d use:%d", pReq->handle, epSet.numOfEps, epSet.inUse); + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + if (strcmp(epSet.eps[i].fqdn, pDnode->localFqdn) == 0 && epSet.eps[i].port == pDnode->serverPort) { + epSet.inUse = (i + 1) % epSet.numOfEps; + } + + epSet.eps[i].port = htons(epSet.eps[i].port); + } + + rpcSendRedirectRsp(pReq->handle, &epSet); +} + +static int32_t dmStart(SMgmtWrapper *pWrapper) { + dDebug("dnode mgmt start to run"); + return dmStartThread(pWrapper->pMgmt); +} + +int32_t dmInit(SMgmtWrapper *pWrapper) { + SDnode *pDnode = pWrapper->pDnode; + SDnodeMgmt *pMgmt = calloc(1, sizeof(SDnodeMgmt)); + dInfo("dnode-mgmt is initialized"); + + pDnode->dnodeId = 0; + pDnode->dropped = 0; + pDnode->clusterId = 0; + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pDnode; + pMgmt->pWrapper = pWrapper; + taosInitRWLatch(&pMgmt->latch); + + pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMgmt->dnodeHash == NULL) { + dError("failed to init dnode hash"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (dmReadFile(pMgmt) != 0) { + dError("failed to read file since %s", terrstr()); + return -1; + } + + if (pDnode->dropped) { + dError("dnode will not start since its already dropped"); + return -1; + } + + if (dmStartWorker(pMgmt) != 0) { + return -1; + } + + pWrapper->pMgmt = pMgmt; + dInfo("dnode-mgmt is initialized"); + return 0; +} + +void dmCleanup(SMgmtWrapper *pWrapper) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("dnode-mgmt start to clean up"); + dmStopWorker(pMgmt); + + taosWLockLatch(&pMgmt->latch); + + if (pMgmt->dnodeEps != NULL) { + taosArrayDestroy(pMgmt->dnodeEps); + pMgmt->dnodeEps = NULL; + } + + if (pMgmt->dnodeHash != NULL) { + taosHashCleanup(pMgmt->dnodeHash); + pMgmt->dnodeHash = NULL; + } + + taosWUnLockLatch(&pMgmt->latch); + + free(pMgmt); + pWrapper->pMgmt = NULL; + dInfo("dnode-mgmt is cleaned up"); +} + +int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { + *required = true; + return 0; +} + +void dmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = dmInit; + mgmtFp.closeFp = dmCleanup; + mgmtFp.startFp = dmStart; + mgmtFp.requiredFp = dmRequire; + + dmInitMsgHandles(pWrapper); + pWrapper->name = "dnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/dnode/src/dmMsg.c b/source/dnode/mgmt/dnode/src/dmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..f451c2ab05669b3903e95571e53aea6e97af35aa --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmMsg.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" +#include "vm.h" + +void dmSendStatusReq(SDnodeMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + SStatusReq req = {0}; + + taosRLockLatch(&pMgmt->latch); + req.sver = tsVersion; + req.dver = pMgmt->dver; + req.dnodeId = pDnode->dnodeId; + req.clusterId = pDnode->clusterId; + req.rebootTime = pDnode->rebootTime; + req.updateTime = pMgmt->updateTime; + req.numOfCores = tsNumOfCores; + req.numOfSupportVnodes = pDnode->numOfSupportVnodes; + tstrncpy(req.dnodeEp, pDnode->localEp, TSDB_EP_LEN); + + req.clusterCfg.statusInterval = tsStatusInterval; + req.clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); + taosRUnLockLatch(&pMgmt->latch); + + req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); + vmMonitorVnodeLoads(dndAcquireWrapper(pDnode, VNODES), req.pVloads); + + int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); + void *pHead = rpcMallocCont(contLen); + tSerializeSStatusReq(pHead, contLen, &req); + taosArrayDestroy(req.pVloads); + + SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; + pMgmt->statusSent = 1; + + dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); + dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg); +} + +static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { + SDnode *pDnode = pMgmt->pDnode; + + if (pDnode->dnodeId == 0) { + dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); + taosWLockLatch(&pMgmt->latch); + pDnode->dnodeId = pCfg->dnodeId; + pDnode->clusterId = pCfg->clusterId; + dmWriteFile(pMgmt); + taosWUnLockLatch(&pMgmt->latch); + } +} + +int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pRsp = &pMsg->rpcMsg; + + if (pRsp->code != TSDB_CODE_SUCCESS) { + if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pDnode->dropped && pDnode->dnodeId > 0) { + dInfo("dnode:%d, set to dropped since not exist in mnode", pDnode->dnodeId); + pDnode->dropped = 1; + dmWriteFile(pMgmt); + } + } else { + SStatusRsp statusRsp = {0}; + if (pRsp->pCont != NULL && pRsp->contLen != 0 && + tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { + pMgmt->dver = statusRsp.dver; + dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); + dmUpdateDnodeEps(pMgmt, statusRsp.pDnodeEps); + } + tFreeSStatusRsp(&statusRsp); + } + + pMgmt->statusSent = 0; +} + +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("auth rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("grant rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SDCfgDnodeReq *pCfg = pReq->pCont; + dError("config req is received, but not supported yet"); + return TSDB_CODE_OPS_NOT_SUPPORT; +} + +void dmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by DNODE + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg); + + // Requests handled by MNODE + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg); +} diff --git a/source/dnode/mgmt/dnode/src/dmWorker.c b/source/dnode/mgmt/dnode/src/dmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..e06cbf43512427a3075622ee18ec6cc3b7e94a04 --- /dev/null +++ b/source/dnode/mgmt/dnode/src/dmWorker.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "bm.h" +#include "dmInt.h" +#include "mm.h" +#include "qm.h" +#include "sm.h" +#include "vm.h" + +static void *dmThreadRoutine(void *param) { + SDnodeMgmt *pMgmt = param; + SDnode *pDnode = pMgmt->pDnode; + int64_t lastStatusTime = taosGetTimestampMs(); + int64_t lastMonitorTime = lastStatusTime; + + setThreadName("dnode-hb"); + + while (true) { + taosThreadTestCancel(); + taosMsleep(200); + if (dndGetStatus(pDnode) != DND_STAT_RUNNING || pDnode->dropped) { + continue; + } + + int64_t curTime = taosGetTimestampMs(); + + float statusInterval = (curTime - lastStatusTime) / 1000.0f; + if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { + dmSendStatusReq(pMgmt); + lastStatusTime = curTime; + } + + float monitorInterval = (curTime - lastMonitorTime) / 1000.0f; + if (monitorInterval >= tsMonitorInterval) { + dndSendMonitorReport(pDnode); + lastMonitorTime = curTime; + } + } +} + +static void dmProcessQueue(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + dTrace("msg:%p, will be processed in dnode queue", pMsg); + + switch (pRpc->msgType) { + case TDMT_DND_CREATE_MNODE: + case TDMT_DND_CREATE_QNODE: + case TDMT_DND_CREATE_SNODE: + case TDMT_DND_CREATE_BNODE: + case TDMT_DND_DROP_MNODE: + case TDMT_DND_DROP_QNODE: + case TDMT_DND_DROP_SNODE: + case TDMT_DND_DROP_BNODE: + code = dndProcessNodeMsg(pMgmt->pDnode, pMsg); + break; + case TDMT_DND_CONFIG_DNODE: + code = dmProcessConfigReq(pMgmt, pMsg); + break; + case TDMT_MND_STATUS_RSP: + code = dmProcessStatusRsp(pMgmt, pMsg); + break; + case TDMT_MND_AUTH_RSP: + code = dmProcessAuthRsp(pMgmt, pMsg); + break; + case TDMT_MND_GRANT_RSP: + code = dmProcessGrantRsp(pMgmt, pMsg); + break; + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType)); + } + + if (pRpc->msgType & 1u) { + if (code != 0) code = terrno; + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; + rpcSendResponse(&rsp); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +int32_t dmStartWorker(SDnodeMgmt *pMgmt) { + if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dmProcessQueue) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); + return -1; + } + + if (dndInitWorker(pMgmt, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dmProcessQueue) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); + return -1; + } + + return 0; +} + +int32_t dmStartThread(SDnodeMgmt *pMgmt) { + pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); + if (pMgmt->threadId == NULL) { + dError("failed to init dnode thread"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +void dmStopWorker(SDnodeMgmt *pMgmt) { + dndCleanupWorker(&pMgmt->mgmtWorker); + dndCleanupWorker(&pMgmt->statusWorker); + + if (pMgmt->threadId != NULL) { + taosDestoryThread(pMgmt->threadId); + pMgmt->threadId = NULL; + } +} + +int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->mgmtWorker; + if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { + pWorker = &pMgmt->statusWorker; + } + + dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt deleted file mode 100644 index c99a4055274f274f87cb030fa07a20263d67f88d..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -aux_source_directory(src DNODE_SRC) -add_library(dnode STATIC ${DNODE_SRC}) -target_link_libraries( - dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor -) -target_include_directories( - dnode - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h deleted file mode 100644 index aeea5386b4ea39d42cac8d599e53d595f71c5660..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_ENV_H_ -#define _TD_DND_ENV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "dndInt.h" - -typedef struct { - EWorkerType type; - const char *name; - int32_t minNum; - int32_t maxNum; - void *queueFp; - SDnode *pDnode; - STaosQueue *queue; - union { - SQWorkerPool pool; - SWWorkerPool mpool; - }; -} SDnodeWorker; - -typedef struct { - char *dnode; - char *mnode; - char *snode; - char *bnode; - char *vnodes; -} SDnodeDir; - -typedef struct { - int32_t dnodeId; - int32_t dropped; - int64_t clusterId; - int64_t dver; - int64_t rebootTime; - int64_t updateTime; - int8_t statusSent; - SEpSet mnodeEpSet; - char *file; - SHashObj *dnodeHash; - SArray *pDnodeEps; - pthread_t *threadId; - SRWLatch latch; - SDnodeWorker mgmtWorker; - SDnodeWorker statusWorker; -} SDnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SMnode *pMnode; - SRWLatch latch; - SDnodeWorker readWorker; - SDnodeWorker writeWorker; - SDnodeWorker syncWorker; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; -} SMnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SQnode *pQnode; - SRWLatch latch; - SDnodeWorker queryWorker; - SDnodeWorker fetchWorker; -} SQnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - int8_t uniqueWorkerInUse; - SSnode *pSnode; - SRWLatch latch; - SArray *uniqueWorkers; // SArray - SDnodeWorker sharedWorker; -} SSnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SBnode *pBnode; - SRWLatch latch; - SDnodeWorker writeWorker; -} SBnodeMgmt; - -typedef struct { - int32_t openVnodes; - int32_t totalVnodes; - int32_t masterNum; - int64_t numOfSelectReqs; - int64_t numOfInsertReqs; - int64_t numOfInsertSuccessReqs; - int64_t numOfBatchInsertReqs; - int64_t numOfBatchInsertSuccessReqs; -} SVnodesStat; - -typedef struct { - SVnodesStat stat; - SHashObj *hash; - SRWLatch latch; - SQWorkerPool queryPool; - SFWorkerPool fetchPool; - SWWorkerPool syncPool; - SWWorkerPool writePool; -} SVnodesMgmt; - -typedef struct { - void *serverRpc; - void *clientRpc; - DndMsgFp msgFp[TDMT_MAX]; -} STransMgmt; - -typedef struct SDnode { - EStat stat; - SDnodeObjCfg cfg; - SDnodeDir dir; - TdFilePtr pLockFile; - SDnodeMgmt dmgmt; - SMnodeMgmt mmgmt; - SQnodeMgmt qmgmt; - SSnodeMgmt smgmt; - SBnodeMgmt bmgmt; - SVnodesMgmt vmgmt; - STransMgmt tmgmt; - STfs *pTfs; - SStartupReq startup; -} SDnode; - -int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_ENV_H_*/ diff --git a/source/dnode/mgmt/impl/inc/dndMgmt.h b/source/dnode/mgmt/impl/inc/dndMgmt.h deleted file mode 100644 index 9cc0c4ae66da13c41c56a298e8a7839fa2a2095f..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndMgmt.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_DNODE_H_ -#define _TD_DND_DNODE_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitMgmt(SDnode *pDnode); -void dndStopMgmt(SDnode *pDnode); -void dndCleanupMgmt(SDnode *pDnode); - -int32_t dndGetDnodeId(SDnode *pDnode); -int64_t dndGetClusterId(SDnode *pDnode); -void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort); -void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet); - -void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg); -void dndSendStatusReq(SDnode *pDnode); -void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet); -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndMnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h deleted file mode 100644 index 0f03bb3832fb7c66bf651bdec18ec6ca196643d9..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndMnode.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_MNODE_H_ -#define _TD_DND_MNODE_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitMnode(SDnode *pDnode); -void dndCleanupMnode(SDnode *pDnode); - -int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); -void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); - -int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h deleted file mode 100644 index 895e94060f741722830f5e71d06c749a2c74dec6..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_VNODES_H_ -#define _TD_DND_VNODES_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitVnodes(SDnode *pDnode); -void dndCleanupVnodes(SDnode *pDnode); -void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads); -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); - -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq); -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq); - -int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pReq); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndWorker.h b/source/dnode/mgmt/impl/inc/dndWorker.h deleted file mode 100644 index 9c037d91c72f02e451dfbf379dc6de4b9ce6a8fc..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/inc/dndWorker.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_WORKER_H_ -#define _TD_DND_WORKER_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "dndEnv.h" - -int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, - int32_t maxNum, void *queueFp); -void dndCleanupWorker(SDnodeWorker *pWorker); -int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_WORKER_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c deleted file mode 100644 index 81b020c1523e8f3443dc1ee9e68c3e52716c4337..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ /dev/null @@ -1,392 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndBnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); - -static SBnode *dndAcquireBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - SBnode *pBnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pBnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pBnode = pMgmt->pBnode; - } else { - terrno = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pBnode != NULL) { - dTrace("acquire bnode, refCount:%d", refCount); - } - return pBnode; -} - -static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) { - if (pBnode == NULL) return; - - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release bnode, refCount:%d", refCount); -} - -static int32_t dndReadBnodeFile(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - int32_t code = TSDB_CODE_DND_BNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_BNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_BNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_BNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_BNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_BNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_BNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteBnodeFile(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartBnodeWorker(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, dndProcessBnodeQueue) != 0) { - dError("failed to start bnode write worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopBnodeWorker(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->writeWorker); -} - -static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode != NULL) { - dndReleaseBnode(pDnode, pBnode); - terrno = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; - dError("failed to create bnode since %s", terrstr()); - return -1; - } - - SBnodeOpt option = {0}; - dndBuildBnodeOption(pDnode, &option); - - pBnode = bndOpen(pDnode->dir.bnode, &option); - if (pBnode == NULL) { - dError("failed to open bnode since %s", terrstr()); - return -1; - } - - if (dndStartBnodeWorker(pDnode) != 0) { - dError("failed to start bnode worker since %s", terrstr()); - bndClose(pBnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteBnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write bnode file since %s", terrstr()); - dndStopBnodeWorker(pDnode); - bndClose(pBnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pBnode = pBnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("bnode open successfully"); - return 0; -} - -static int32_t dndDropBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode == NULL) { - dError("failed to drop bnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteBnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseBnode(pDnode, pBnode); - dError("failed to drop bnode since %s", terrstr()); - return -1; - } - - dndReleaseBnode(pDnode, pBnode); - dndStopBnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteBnodeFile(pDnode); - bndClose(pBnode); - pMgmt->pBnode = NULL; - bndDestroy(pDnode->dir.bnode); - - return 0; -} - -int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateBnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION; - dError("failed to create bnode since %s", terrstr()); - return -1; - } else { - return dndOpenBnode(pDnode); - } -} - -int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropBnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION; - dError("failed to drop bnode since %s", terrstr()); - return -1; - } else { - return dndDropBnode(pDnode); - } -} - -static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dndSendBnodeErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - dndSendBnodeErrorRsp(pMsg, code); - } -} - -static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode == NULL) { - dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); - return; - } - - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - if (pArray == NULL) { - dndReleaseBnode(pDnode, pBnode); - dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); - return; - } - - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - void *ptr = taosArrayPush(pArray, &pMsg); - if (ptr == NULL) { - dndSendBnodeErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); - } - } - - bndProcessWMsgs(pBnode, pArray); - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - taosArrayDestroy(pArray); - dndReleaseBnode(pDnode, pBnode); -} - -static void dndWriteBnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; - - SBnode *pBnode = dndAcquireBnode(pDnode); - if (pBnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseBnode(pDnode, pBnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteBnodeMsgToWorker(pDnode, &pDnode->bmgmt.writeWorker, pMsg); -} - -int32_t dndInitBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadBnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("bnode has been deployed and needs to be deleted"); - bndDestroy(pDnode->dir.bnode); - return 0; - } - - if (!pMgmt->deployed) return 0; - - return dndOpenBnode(pDnode); -} - -void dndCleanupBnode(SDnode *pDnode) { - SBnodeMgmt *pMgmt = &pDnode->bmgmt; - if (pMgmt->pBnode) { - dndStopBnodeWorker(pDnode); - bndClose(pMgmt->pBnode); - pMgmt->pBnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c deleted file mode 100644 index 84b2dca3264ec6196ede30e99f6f9c08c2efcacc..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndBnode.h" -#include "dndMgmt.h" -#include "dndMnode.h" -#include "dndQnode.h" -#include "dndSnode.h" -#include "dndTransport.h" -#include "dndVnodes.h" -#include "monitor.h" -#include "sync.h" -#include "tfs.h" -#include "wal.h" - -static int8_t once = DND_ENV_INIT; - -EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } - -void dndSetStat(SDnode *pDnode, EStat stat) { - dDebug("dnode status set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat)); - pDnode->stat = stat; -} - -const char *dndStatStr(EStat stat) { - switch (stat) { - case DND_STAT_INIT: - return "init"; - case DND_STAT_RUNNING: - return "running"; - case DND_STAT_STOPPED: - return "stopped"; - default: - return "unknown"; - } -} - -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); -} - -static TdFilePtr dndCheckRunning(char *dataDir) { - char filepath[PATH_MAX] = {0}; - snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); - - TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - return NULL; - } - - int32_t ret = taosLockFile(pFile); - if (ret != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - taosCloseFile(&pFile); - return NULL; - } - - return pFile; -} - -static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) { - pDnode->pLockFile = dndCheckRunning(pCfg->dataDir); - if (pDnode->pLockFile == NULL) { - return -1; - } - - char path[PATH_MAX + 100]; - snprintf(path, sizeof(path), "%s%smnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.mnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%svnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.vnodes = tstrdup(path); - snprintf(path, sizeof(path), "%s%sdnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.dnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%ssnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.snode = tstrdup(path); - snprintf(path, sizeof(path), "%s%sbnode", pCfg->dataDir, TD_DIRSEP); - pDnode->dir.bnode = tstrdup(path); - - if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL || - pDnode->dir.snode == NULL || pDnode->dir.bnode == NULL) { - dError("failed to malloc dir object"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (taosMkDir(pDnode->dir.dnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.dnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.mnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.mnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.vnodes) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.vnodes, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.snode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.snode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosMkDir(pDnode->dir.bnode) != 0) { - dError("failed to create dir:%s since %s", pDnode->dir.bnode, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); - return 0; -} - -static void dndCloseDir(SDnode *pDnode) { - tfree(pDnode->dir.mnode); - tfree(pDnode->dir.vnodes); - tfree(pDnode->dir.dnode); - tfree(pDnode->dir.snode); - tfree(pDnode->dir.bnode); - - if (pDnode->pLockFile != NULL) { - taosUnLockFile(pDnode->pLockFile); - taosCloseFile(&pDnode->pLockFile); - pDnode->pLockFile = NULL; - } -} - -SDnode *dndCreate(SDnodeObjCfg *pCfg) { - dInfo("start to create dnode object"); - - SDnode *pDnode = calloc(1, sizeof(SDnode)); - if (pDnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - dError("failed to create dnode object since %s", terrstr()); - return NULL; - } - - dndSetStat(pDnode, DND_STAT_INIT); - - if (dndInitDir(pDnode, pCfg) != 0) { - dError("failed to init dnode dir since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - SDiskCfg dCfg = {0}; - tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN); - dCfg.level = 0; - dCfg.primary = 1; - SDiskCfg *pDisks = pDnode->cfg.pDisks; - int32_t numOfDisks = pDnode->cfg.numOfDisks; - if (numOfDisks <= 0 || pDisks == NULL) { - pDisks = &dCfg; - numOfDisks = 1; - } - - pDnode->pTfs = tfsOpen(pDisks, numOfDisks); - if (pDnode->pTfs == NULL) { - dError("failed to init tfs since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitMgmt(pDnode) != 0) { - dError("failed to init mgmt since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitVnodes(pDnode) != 0) { - dError("failed to init vnodes since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitQnode(pDnode) != 0) { - dError("failed to init qnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitSnode(pDnode) != 0) { - dError("failed to init snode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitBnode(pDnode) != 0) { - dError("failed to init bnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitMnode(pDnode) != 0) { - dError("failed to init mnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (dndInitTrans(pDnode) != 0) { - dError("failed to init transport since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - dndSetStat(pDnode, DND_STAT_RUNNING); - dndSendStatusReq(pDnode); - dndReportStartup(pDnode, "TDengine", "initialized successfully"); - dInfo("dnode object is created, data:%p", pDnode); - - return pDnode; -} - -void dndClose(SDnode *pDnode) { - if (pDnode == NULL) return; - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("dnode is shutting down, data:%p", pDnode); - return; - } - - dInfo("start to close dnode, data:%p", pDnode); - dndSetStat(pDnode, DND_STAT_STOPPED); - dndCleanupTrans(pDnode); - dndStopMgmt(pDnode); - dndCleanupMnode(pDnode); - dndCleanupBnode(pDnode); - dndCleanupSnode(pDnode); - dndCleanupQnode(pDnode); - dndCleanupVnodes(pDnode); - dndCleanupMgmt(pDnode); - tfsClose(pDnode->pTfs); - - dndCloseDir(pDnode); - free(pDnode); - dInfo("dnode object is closed, data:%p", pDnode); -} - -int32_t dndInit() { - if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { - terrno = TSDB_CODE_REPEAT_INIT; - dError("failed to init dnode env since %s", terrstr()); - return -1; - } - - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); - - if (rpcInit() != 0) { - dError("failed to init rpc since %s", terrstr()); - dndCleanup(); - return -1; - } - - if (walInit() != 0) { - dError("failed to init wal since %s", terrstr()); - dndCleanup(); - return -1; - } - - SVnodeOpt vnodeOpt = { - .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode}; - - if (vnodeInit(&vnodeOpt) != 0) { - dError("failed to init vnode since %s", terrstr()); - dndCleanup(); - return -1; - } - - SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn, .comp = tsMonitorComp}; - if (monInit(&monCfg) != 0) { - dError("failed to init monitor since %s", terrstr()); - dndCleanup(); - return -1; - } - - dInfo("dnode env is initialized"); - return 0; -} - -void dndCleanup() { - if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { - dError("dnode env is already cleaned up"); - return; - } - - walCleanUp(); - vnodeCleanup(); - rpcCleanup(); - monCleanup(); - - taosStopCacheRefreshWorker(); - dInfo("dnode env is cleaned up"); -} - -int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { - tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); - pInfo->logdir.size = tsLogSpace.size; - tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); - pInfo->tempdir.size = tsTempSpace.size; - - return tfsGetMonitorInfo(pDnode->pTfs, pInfo); -} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c deleted file mode 100644 index 866076a3d0c5b19710ff1c1b6e68ff8285107b31..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ /dev/null @@ -1,763 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndMgmt.h" -#include "dndBnode.h" -#include "dndMnode.h" -#include "dndQnode.h" -#include "dndSnode.h" -#include "dndTransport.h" -#include "dndVnodes.h" -#include "dndWorker.h" -#include "monitor.h" - -static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static int32_t dndReadDnodes(SDnode *pDnode); -static int32_t dndWriteDnodes(SDnode *pDnode); -static void *dnodeThreadRoutine(void *param); - -static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq); -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp); -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pRsp); -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pRsp); - -int32_t dndGetDnodeId(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t dnodeId = pMgmt->dnodeId; - taosRUnLockLatch(&pMgmt->latch); - return dnodeId; -} - -int64_t dndGetClusterId(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - int64_t clusterId = pMgmt->clusterId; - taosRUnLockLatch(&pMgmt->latch); - return clusterId; -} - -void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - - SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); - if (pDnodeEp != NULL) { - if (pPort != NULL) { - *pPort = pDnodeEp->ep.port; - } - if (pFqdn != NULL) { - tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); - } - if (pEp != NULL) { - snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); - } - } - - taosRUnLockLatch(&pMgmt->latch); -} - -void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - *pEpSet = pMgmt->mnodeEpSet; - taosRUnLockLatch(&pMgmt->latch); -} - -void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) { - tmsg_t msgType = pReq->msgType; - - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - - dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - if (strcmp(epSet.eps[i].fqdn, pDnode->cfg.localFqdn) == 0 && epSet.eps[i].port == pDnode->cfg.serverPort) { - epSet.inUse = (i + 1) % epSet.numOfEps; - } - - epSet.eps[i].port = htons(epSet.eps[i].port); - } - - rpcSendRedirectRsp(pReq->handle, &epSet); -} - -static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - pMgmt->mnodeEpSet = *pEpSet; - for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } - - taosWUnLockLatch(&pMgmt->latch); -} - -static void dndPrintDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - dDebug("print dnode ep list, num:%d", numOfEps); - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pEp = taosArrayGet(pMgmt->pDnodeEps, i); - dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); - } -} - -static void dndResetDnodes(SDnode *pDnode, SArray *pDnodeEps) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pMgmt->pDnodeEps != pDnodeEps) { - SArray *tmp = pMgmt->pDnodeEps; - pMgmt->pDnodeEps = taosArrayDup(pDnodeEps); - taosArrayDestroy(tmp); - } - - pMgmt->mnodeEpSet.inUse = 0; - pMgmt->mnodeEpSet.numOfEps = 0; - - int32_t mIndex = 0; - int32_t numOfEps = (int32_t)taosArrayGetSize(pDnodeEps); - - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i); - if (!pDnodeEp->isMnode) continue; - if (mIndex >= TSDB_MAX_REPLICA) continue; - pMgmt->mnodeEpSet.numOfEps++; - - pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep; - mIndex++; - } - - for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i); - taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); - } - - dndPrintDnodes(pDnode); -} - -static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) { - bool changed = false; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - - SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); - if (pDnodeEp != NULL) { - char epstr[TSDB_EP_LEN + 1]; - snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); - changed = strcmp(pEp, epstr) != 0; - } - - taosRUnLockLatch(&pMgmt->latch); - return changed; -} - -static int32_t dndReadDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - pMgmt->pDnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); - if (pMgmt->pDnodeEps == NULL) { - dError("failed to calloc dnodeEp array since %s", strerror(errno)); - goto PRASE_DNODE_OVER; - } - - int32_t code = TSDB_CODE_DND_DNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - // fp = fopen(pMgmt->file, "r"); - TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", pMgmt->file); - code = 0; - goto PRASE_DNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->dnodeId = dnodeId->valueint; - - cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); - if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read %s since clusterId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->clusterId = atoll(clusterId->valuestring); - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); - if (!dnodes || dnodes->type != cJSON_Array) { - dError("failed to read %s since dnodes not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - int32_t numOfDnodes = cJSON_GetArraySize(dnodes); - if (numOfDnodes <= 0) { - dError("failed to read %s since numOfDnodes:%d invalid", pMgmt->file, numOfDnodes); - goto PRASE_DNODE_OVER; - } - - for (int32_t i = 0; i < numOfDnodes; ++i) { - cJSON *node = cJSON_GetArrayItem(dnodes, i); - if (node == NULL) break; - - SDnodeEp dnodeEp = {0}; - - cJSON *did = cJSON_GetObjectItem(node, "id"); - if (!did || did->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - dnodeEp.id = dnodeId->valueint; - - cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { - dError("failed to read %s since dnodeFqdn not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); - if (!dnodePort || dnodePort->type != cJSON_Number) { - dError("failed to read %s since dnodePort not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - - dnodeEp.ep.port = dnodePort->valueint; - - cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); - if (!isMnode || isMnode->type != cJSON_Number) { - dError("failed to read %s since isMnode not found", pMgmt->file); - goto PRASE_DNODE_OVER; - } - dnodeEp.isMnode = isMnode->valueint; - - taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); - } - - code = 0; - dInfo("succcessed to read file %s", pMgmt->file); - dndPrintDnodes(pDnode); - -PRASE_DNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { - dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); - return -1; - } - - if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) { - SDnodeEp dnodeEp = {0}; - dnodeEp.isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep); - taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); - } - - dndResetDnodes(pDnode, pMgmt->pDnodeEps); - - terrno = 0; - return 0; -} - -static int32_t dndWriteDnodes(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - // FILE *fp = fopen(pMgmt->file, "w"); - TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - dError("failed to write %s since %s", pMgmt->file, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pMgmt->dnodeId); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pMgmt->clusterId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); - - int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - for (int32_t i = 0; i < numOfEps; ++i) { - SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->pDnodeEps, i); - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); - len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); - if (i < numOfEps - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - terrno = 0; - - pMgmt->updateTime = taosGetTimestampMs(); - dDebug("successed to write %s", pMgmt->file); - return 0; -} - -void dndSendStatusReq(SDnode *pDnode) { - SStatusReq req = {0}; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosRLockLatch(&pMgmt->latch); - req.sver = tsVersion; - req.dver = pMgmt->dver; - req.dnodeId = pMgmt->dnodeId; - req.clusterId = pMgmt->clusterId; - req.rebootTime = pMgmt->rebootTime; - req.updateTime = pMgmt->updateTime; - req.numOfCores = tsNumOfCores; - req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes; - memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); - - req.clusterCfg.statusInterval = tsStatusInterval; - req.clusterCfg.checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); - memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); - taosRUnLockLatch(&pMgmt->latch); - - req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); - dndGetVnodeLoads(pDnode, req.pVloads); - - int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); - tSerializeSStatusReq(pHead, contLen, &req); - taosArrayDestroy(req.pVloads); - - SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; - pMgmt->statusSent = 1; - - dTrace("pDnode:%p, send status req to mnode", pDnode); - dndSendReqToMnode(pDnode, &rpcMsg); -} - -static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - if (pMgmt->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); - taosWLockLatch(&pMgmt->latch); - pMgmt->dnodeId = pCfg->dnodeId; - pMgmt->clusterId = pCfg->clusterId; - dndWriteDnodes(pDnode); - taosWUnLockLatch(&pMgmt->latch); - } -} - -static void dndUpdateDnodeEps(SDnode *pDnode, SArray *pDnodeEps) { - int32_t numOfEps = taosArrayGetSize(pDnodeEps); - if (numOfEps <= 0) return; - - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps); - if (numOfEps != numOfEpsOld) { - dndResetDnodes(pDnode, pDnodeEps); - dndWriteDnodes(pDnode); - } else { - int32_t size = numOfEps * sizeof(SDnodeEp); - if (memcmp(pMgmt->pDnodeEps->pData, pDnodeEps->pData, size) != 0) { - dndResetDnodes(pDnode, pDnodeEps); - dndWriteDnodes(pDnode); - } - } - - taosWUnLockLatch(&pMgmt->latch); -} - -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pRsp->code != TSDB_CODE_SUCCESS) { - if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) { - dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->dnodeId); - pMgmt->dropped = 1; - dndWriteDnodes(pDnode); - } - } else { - SStatusRsp statusRsp = {0}; - if (pRsp->pCont != NULL && pRsp->contLen != 0 && - tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { - pMgmt->dver = statusRsp.dver; - dndUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg); - dndUpdateDnodeEps(pDnode, statusRsp.pDnodeEps); - } - taosArrayDestroy(statusRsp.pDnodeEps); - } - - pMgmt->statusSent = 0; -} - -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pReq) { dError("auth rsp is received, but not supported yet"); } - -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pReq) { - dError("grant rsp is received, but not supported yet"); -} - -static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - dError("config req is received, but not supported yet"); - SDCfgDnodeReq *pCfg = pReq->pCont; - return TSDB_CODE_OPS_NOT_SUPPORT; -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - - SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; - rpcSendResponse(&rpcRsp); -} - -static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { - pInfo->dnode_id = dndGetDnodeId(pDnode); - tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); - pInfo->cluster_id = dndGetClusterId(pDnode); - pInfo->protocol = 1; -} - -static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f); - taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); - pInfo->cpu_cores = tsNumOfCores; - taosGetProcMemory(&pInfo->mem_engine); - taosGetSysMemory(&pInfo->mem_system); - pInfo->mem_total = tsTotalMemoryKB; - pInfo->disk_engine = 0; - pInfo->disk_used = tsDataSpace.size.used; - pInfo->disk_total = tsDataSpace.size.total; - taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); - taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); - - SVnodesStat *pStat = &pDnode->vmgmt.stat; - pInfo->req_select = pStat->numOfSelectReqs; - pInfo->req_insert = pStat->numOfInsertReqs; - pInfo->req_insert_success = pStat->numOfInsertSuccessReqs; - pInfo->req_insert_batch = pStat->numOfBatchInsertReqs; - pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs; - pInfo->errors = tsNumOfErrorLogs; - pInfo->vnodes_num = pStat->totalVnodes; - pInfo->masters = pStat->masterNum; - pInfo->has_mnode = pDnode->mmgmt.deployed; -} - -static void dndSendMonitorReport(SDnode *pDnode) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort); - - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - - SMonBasicInfo basicInfo = {0}; - dndGetMonitorBasicInfo(pDnode, &basicInfo); - monSetBasicInfo(pMonitor, &basicInfo); - - SMonClusterInfo clusterInfo = {0}; - SMonVgroupInfo vgroupInfo = {0}; - SMonGrantInfo grantInfo = {0}; - if (dndGetMnodeMonitorInfo(pDnode, &clusterInfo, &vgroupInfo, &grantInfo) == 0) { - monSetClusterInfo(pMonitor, &clusterInfo); - monSetVgroupInfo(pMonitor, &vgroupInfo); - monSetGrantInfo(pMonitor, &grantInfo); - } - - SMonDnodeInfo dnodeInfo = {0}; - dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); - monSetDnodeInfo(pMonitor, &dnodeInfo); - - SMonDiskInfo diskInfo = {0}; - if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { - monSetDiskInfo(pMonitor, &diskInfo); - } - - taosArrayDestroy(clusterInfo.dnodes); - taosArrayDestroy(clusterInfo.mnodes); - taosArrayDestroy(vgroupInfo.vgroups); - taosArrayDestroy(diskInfo.datadirs); - - monSendReport(pMonitor); - monCleanupMonitorInfo(pMonitor); -} - -static void *dnodeThreadRoutine(void *param) { - SDnode *pDnode = param; - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int64_t lastStatusTime = taosGetTimestampMs(); - int64_t lastMonitorTime = lastStatusTime; - - setThreadName("dnode-hb"); - - while (true) { - pthread_testcancel(); - taosMsleep(200); - if (dndGetStat(pDnode) != DND_STAT_RUNNING || pMgmt->dropped) { - continue; - } - - int64_t curTime = taosGetTimestampMs(); - - float statusInterval = (curTime - lastStatusTime) / 1000.0f; - if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { - dndSendStatusReq(pDnode); - lastStatusTime = curTime; - } - - float monitorInterval = (curTime - lastMonitorTime) / 1000.0f; - if (monitorInterval >= tsMonitorInterval) { - dndSendMonitorReport(pDnode); - lastMonitorTime = curTime; - } - } -} - -int32_t dndInitMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - pMgmt->dnodeId = 0; - pMgmt->rebootTime = taosGetTimestampMs(); - pMgmt->dropped = 0; - pMgmt->clusterId = 0; - taosInitRWLatch(&pMgmt->latch); - - char path[PATH_MAX]; - snprintf(path, PATH_MAX, "%s/dnode.json", pDnode->dir.dnode); - pMgmt->file = strdup(path); - if (pMgmt->file == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - if (pMgmt->dnodeHash == NULL) { - dError("failed to init dnode hash"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (dndReadDnodes(pDnode) != 0) { - dError("failed to read file:%s since %s", pMgmt->file, terrstr()); - return -1; - } - - if (pMgmt->dropped) { - dError("dnode not start since its already dropped"); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dndProcessMgmtQueue) != 0) { - dError("failed to start dnode mgmt worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dndProcessMgmtQueue) != 0) { - dError("failed to start dnode mgmt worker since %s", terrstr()); - return -1; - } - - pMgmt->threadId = taosCreateThread(dnodeThreadRoutine, pDnode); - if (pMgmt->threadId == NULL) { - dError("failed to init dnode thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dInfo("dnode-mgmt is initialized"); - return 0; -} - -void dndStopMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - dndCleanupWorker(&pMgmt->mgmtWorker); - dndCleanupWorker(&pMgmt->statusWorker); - - if (pMgmt->threadId != NULL) { - taosDestoryThread(pMgmt->threadId); - pMgmt->threadId = NULL; - } -} - -void dndCleanupMgmt(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - taosWLockLatch(&pMgmt->latch); - - if (pMgmt->pDnodeEps != NULL) { - taosArrayDestroy(pMgmt->pDnodeEps); - pMgmt->pDnodeEps = NULL; - } - - if (pMgmt->dnodeHash != NULL) { - taosHashCleanup(pMgmt->dnodeHash); - pMgmt->dnodeHash = NULL; - } - - if (pMgmt->file != NULL) { - free(pMgmt->file); - pMgmt->file = NULL; - } - - taosWUnLockLatch(&pMgmt->latch); - dInfo("dnode-mgmt is cleaned up"); -} - -void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - - if (pEpSet && pEpSet->numOfEps > 0 && pMsg->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pDnode, pEpSet); - } - - SDnodeWorker *pWorker = &pMgmt->mgmtWorker; - if (pMsg->msgType == TDMT_MND_STATUS_RSP) { - pWorker = &pMgmt->statusWorker; - } - - if (dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)) != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } -} - -static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t code = 0; - - switch (pMsg->msgType) { - case TDMT_DND_CREATE_MNODE: - code = dndProcessCreateMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_ALTER_MNODE: - code = dndProcessAlterMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_MNODE: - code = dndProcessDropMnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_QNODE: - code = dndProcessCreateQnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_QNODE: - code = dndProcessDropQnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_SNODE: - code = dndProcessCreateSnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_SNODE: - code = dndProcessDropSnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CREATE_BNODE: - code = dndProcessCreateBnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_BNODE: - code = dndProcessDropBnodeReq(pDnode, pMsg); - break; - case TDMT_DND_CONFIG_DNODE: - code = dndProcessConfigDnodeReq(pDnode, pMsg); - break; - case TDMT_MND_STATUS_RSP: - dndProcessStatusRsp(pDnode, pMsg); - break; - case TDMT_MND_AUTH_RSP: - dndProcessAuthRsp(pDnode, pMsg); - break; - case TDMT_MND_GRANT_RSP: - dndProcessGrantRsp(pDnode, pMsg); - break; - case TDMT_DND_CREATE_VNODE: - code = dndProcessCreateVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_ALTER_VNODE: - code = dndProcessAlterVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_DROP_VNODE: - code = dndProcessDropVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_SYNC_VNODE: - code = dndProcessSyncVnodeReq(pDnode, pMsg); - break; - case TDMT_DND_COMPACT_VNODE: - code = dndProcessCompactVnodeReq(pDnode, pMsg); - break; - default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - code = -1; - dError("RPC %p, dnode msg:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType)); - break; - } - - if (pMsg->msgType & 1u) { - if (code != 0) code = terrno; - SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle}; - rpcSendResponse(&rsp); - } - - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; - taosFreeQitem(pMsg); -} diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c deleted file mode 100644 index 0ea47c89d8cfa6af4307001a753fa0bfb00b09a2..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ /dev/null @@ -1,642 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndMnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg); -static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg); - -static SMnode *dndAcquireMnode(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - SMnode *pMnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pMnode = pMgmt->pMnode; - } else { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pMnode != NULL) { - dTrace("acquire mnode, refCount:%d", refCount); - } - return pMnode; -} - -static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) { - if (pMnode == NULL) return; - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release mnode, refCount:%d", refCount); -} - -static int32_t dndReadMnodeFile(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - int32_t code = TSDB_CODE_DND_MNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_MNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_MNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_MNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_MNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_MNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); - if (!mnodes || mnodes->type != cJSON_Array) { - dError("failed to read %s since nodes not found", file); - goto PRASE_MNODE_OVER; - } - - pMgmt->replica = cJSON_GetArraySize(mnodes); - if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { - dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); - goto PRASE_MNODE_OVER; - } - - for (int32_t i = 0; i < pMgmt->replica; ++i) { - cJSON *node = cJSON_GetArrayItem(mnodes, i); - if (node == NULL) break; - - SReplica *pReplica = &pMgmt->replicas[i]; - - cJSON *id = cJSON_GetObjectItem(node, "id"); - if (!id || id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); - goto PRASE_MNODE_OVER; - } - pReplica->id = id->valueint; - - cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); - goto PRASE_MNODE_OVER; - } - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *port = cJSON_GetObjectItem(node, "port"); - if (!port || port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); - goto PRASE_MNODE_OVER; - } - pReplica->port = port->valueint; - } - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_MNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteMnodeFile(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); - for (int32_t i = 0; i < pMgmt->replica; ++i) { - SReplica *pReplica = &pMgmt->replicas[i]; - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); - if (i < pMgmt->replica - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartMnodeWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - if (dndInitWorker(pDnode, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode read worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode write worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, dndProcessMnodeQueue) != 0) { - dError("failed to start mnode sync worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopMnodeWorker(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 1) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->readWorker); - dndCleanupWorker(&pMgmt->writeWorker); - dndCleanupWorker(&pMgmt->syncWorker); -} - -static bool dndNeedDeployMnode(SDnode *pDnode) { - if (dndGetDnodeId(pDnode) > 0) { - return false; - } - - if (dndGetClusterId(pDnode) > 0) { - return false; - } - - if (strcmp(pDnode->cfg.localEp, pDnode->cfg.firstEp) != 0) { - return false; - } - - return true; -} - -static int32_t dndPutMsgToMWriteQ(SDnode *pDnode, SRpcMsg *pRpcMsg) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pRpcMsg); - return 0; -} - -static int32_t dndPutMsgToMReadQ(SDnode *pDnode, SRpcMsg* pRpcMsg) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pRpcMsg); - return 0; -} - -static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->putReqToMWriteQFp = dndPutMsgToMWriteQ; - pOption->putReqToMReadQFp = dndPutMsgToMReadQ; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); -} - -static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { - dndInitMnodeOption(pDnode, pOption); - pOption->replica = 1; - pOption->selfIndex = 0; - SReplica *pReplica = &pOption->replicas[0]; - pReplica->id = 1; - pReplica->port = pDnode->cfg.serverPort; - memcpy(pReplica->fqdn, pDnode->cfg.localFqdn, TSDB_FQDN_LEN); - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->selfIndex = pOption->selfIndex; - pMgmt->replica = pOption->replica; - memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); -} - -static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) { - dndInitMnodeOption(pDnode, pOption); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pOption->selfIndex = pMgmt->selfIndex; - pOption->replica = pMgmt->replica; - memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); -} - -static int32_t dndBuildMnodeOptionFromReq(SDnode *pDnode, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) { - dndInitMnodeOption(pDnode, pOption); - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - - pOption->replica = pCreate->replica; - pOption->selfIndex = -1; - for (int32_t i = 0; i < pCreate->replica; ++i) { - SReplica *pReplica = &pOption->replicas[i]; - pReplica->id = pCreate->replicas[i].id; - pReplica->port = pCreate->replicas[i].port; - memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); - if (pReplica->id == pOption->dnodeId) { - pOption->selfIndex = i; - } - } - - if (pOption->selfIndex == -1) { - dError("failed to build mnode options since %s", terrstr()); - return -1; - } - - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - pMgmt->selfIndex = pOption->selfIndex; - pMgmt->replica = pOption->replica; - memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); - return 0; -} - -static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = mndOpen(pDnode->dir.mnode, pOption); - if (pMnode == NULL) { - dError("failed to open mnode since %s", terrstr()); - return -1; - } - - if (dndStartMnodeWorker(pDnode) != 0) { - dError("failed to start mnode worker since %s", terrstr()); - mndClose(pMnode); - mndDestroy(pDnode->dir.mnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteMnodeFile(pDnode) != 0) { - dError("failed to write mnode file since %s", terrstr()); - pMgmt->deployed = 0; - dndStopMnodeWorker(pDnode); - mndClose(pMnode); - mndDestroy(pDnode->dir.mnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pMnode = pMnode; - pMgmt->deployed = 1; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("mnode open successfully"); - return 0; -} - -static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOpt *pOption) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - if (mndAlter(pMnode, pOption) != 0) { - dError("failed to alter mnode since %s", terrstr()); - dndReleaseMnode(pDnode, pMnode); - return -1; - } - - dndReleaseMnode(pDnode, pMnode); - return 0; -} - -static int32_t dndDropMnode(SDnode *pDnode) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteMnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseMnode(pDnode, pMnode); - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - dndReleaseMnode(pDnode, pMnode); - dndStopMnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteMnodeFile(pDnode); - mndClose(pMnode); - pMgmt->pMnode = NULL; - mndDestroy(pDnode->dir.mnode); - - return 0; -} - - -int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateMnodeReq createReq = {0}; - if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.replica <= 1 || createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - SMnodeOpt option = {0}; - if (dndBuildMnodeOptionFromReq(pDnode, &option, &createReq) != 0) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - dndReleaseMnode(pDnode, pMnode); - terrno = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; - dError("failed to create mnode since %s", terrstr()); - return -1; - } - - dDebug("start to create mnode"); - return dndOpenMnode(pDnode, &option); -} - -int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDAlterMnodeReq alterReq = {0}; - if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (alterReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - SMnodeOpt option = {0}; - if (dndBuildMnodeOptionFromReq(pDnode, &option, &alterReq) != 0) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - dError("failed to alter mnode since %s", terrstr()); - return -1; - } - - dDebug("start to alter mnode"); - int32_t code = dndAlterMnode(pDnode, &option); - dndReleaseMnode(pDnode, pMnode); - - return code; -} - -int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropMnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - dError("failed to drop mnode since %s", terrstr()); - return -1; - } - - dDebug("start to drop mnode"); - int32_t code = dndDropMnode(pDnode); - dndReleaseMnode(pDnode, pMnode); - - return code; -} - -static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - mndProcessMsg(pMsg); - dndReleaseMnode(pDnode, pMnode); - } else { - mndSendRsp(pMsg, terrno); - } - - mndCleanupMsg(pMsg); -} - -static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg) { - int32_t code = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode != NULL) { - SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - code = dndWriteMsgToWorker(pWorker, pMsg, 0); - if (code != 0) code = terrno; - } - - if (code != 0) { - mndCleanupMsg(pMsg); - } - } - dndReleaseMnode(pDnode, pMnode); - - if (code != 0) { - if (pRpcMsg->msgType & 1u) { - if (code == TSDB_CODE_DND_MNODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) { - dndSendRedirectRsp(pDnode, pRpcMsg); - } else { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .ahandle = pRpcMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - } - rpcFreeCont(pRpcMsg->pCont); - } -} - -void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pMsg); -} - -void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.syncWorker, pMsg); -} - -void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pMsg); -} - -int32_t dndInitMnode(SDnode *pDnode) { - dInfo("dnode-mnode start to init"); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadMnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("mnode has been deployed and needs to be deleted"); - mndDestroy(pDnode->dir.mnode); - return 0; - } - - if (!pMgmt->deployed) { - bool needDeploy = dndNeedDeployMnode(pDnode); - if (!needDeploy) { - dDebug("mnode does not need to be deployed"); - return 0; - } - - dInfo("start to deploy mnode"); - SMnodeOpt option = {0}; - dndBuildMnodeDeployOption(pDnode, &option); - return dndOpenMnode(pDnode, &option); - } else { - dInfo("start to open mnode"); - SMnodeOpt option = {0}; - dndBuildMnodeOpenOption(pDnode, &option); - return dndOpenMnode(pDnode, &option); - } -} - -void dndCleanupMnode(SDnode *pDnode) { - dInfo("dnode-mnode start to clean up"); - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - if (pMgmt->pMnode) { - dndStopMnodeWorker(pDnode); - mndClose(pMgmt->pMnode); - pMgmt->pMnode = NULL; - } - dInfo("dnode-mnode is cleaned up"); -} - -int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SMnodeMgmt *pMgmt = &pDnode->mmgmt; - - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) { - terrno = TSDB_CODE_APP_NOT_READY; - dTrace("failed to get user auth since %s", terrstr()); - return -1; - } - - int32_t code = mndRetriveAuth(pMnode, user, spi, encrypt, secret, ckey); - dndReleaseMnode(pDnode, pMnode); - - dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); - return code; -} - -int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { - SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL) return -1; - - int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); - dndReleaseMnode(pDnode, pMnode); - return code; -} diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c deleted file mode 100644 index 93e2209610b98e957b956c5f9caa6babe8004e32..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndQnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static SQnode *dndAcquireQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SQnode *pQnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pQnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pQnode = pMgmt->pQnode; - } else { - terrno = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pQnode != NULL) { - dTrace("acquire qnode, refCount:%d", refCount); - } - return pQnode; -} - -static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode) { - if (pQnode == NULL) return; - - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release qnode, refCount:%d", refCount); -} - -static int32_t dndReadQnodeFile(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - int32_t code = TSDB_CODE_DND_QNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_QNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_QNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_QNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_QNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_QNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_QNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteQnodeFile(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartQnodeWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, dndProcessQnodeQueue) != 0) { - dError("failed to start qnode query worker since %s", terrstr()); - return -1; - } - - if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, dndProcessQnodeQueue) != 0) { - dError("failed to start qnode fetch worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopQnodeWorker(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - dndCleanupWorker(&pMgmt->queryWorker); - dndCleanupWorker(&pMgmt->fetchWorker); -} - -static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - dndReleaseQnode(pDnode, pQnode); - terrno = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; - dError("failed to create qnode since %s", terrstr()); - return -1; - } - - SQnodeOpt option = {0}; - dndBuildQnodeOption(pDnode, &option); - - pQnode = qndOpen(&option); - if (pQnode == NULL) { - dError("failed to open qnode since %s", terrstr()); - return -1; - } - - if (dndStartQnodeWorker(pDnode) != 0) { - dError("failed to start qnode worker since %s", terrstr()); - qndClose(pQnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteQnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write qnode file since %s", terrstr()); - dndStopQnodeWorker(pDnode); - qndClose(pQnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pQnode = pQnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("qnode open successfully"); - return 0; -} - -static int32_t dndDropQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode == NULL) { - dError("failed to drop qnode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteQnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseQnode(pDnode, pQnode); - dError("failed to drop qnode since %s", terrstr()); - return -1; - } - - dndReleaseQnode(pDnode, pQnode); - dndStopQnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteQnodeFile(pDnode); - qndClose(pQnode); - pMgmt->pQnode = NULL; - - return 0; -} - -int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateQnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION; - dError("failed to create qnode since %s", terrstr()); - return -1; - } else { - return dndOpenQnode(pDnode); - } -} - -int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropQnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION; - dError("failed to drop qnode since %s", terrstr()); - return -1; - } else { - return dndDropQnode(pDnode); - } -} - -static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - SRpcMsg *pRsp = NULL; - int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - code = qndProcessMsg(pQnode, pMsg, &pRsp); - } - dndReleaseQnode(pDnode, pQnode); - - if (pMsg->msgType & 1u) { - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dndWriteQnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; - - SQnode *pQnode = dndAcquireQnode(pDnode); - if (pQnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseQnode(pDnode, pQnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg); -} - -void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg); -} - -int32_t dndInitQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadQnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) return 0; - if (!pMgmt->deployed) return 0; - - return dndOpenQnode(pDnode); -} - -void dndCleanupQnode(SDnode *pDnode) { - SQnodeMgmt *pMgmt = &pDnode->qmgmt; - if (pMgmt->pQnode) { - dndStopQnodeWorker(pDnode); - qndClose(pMgmt->pQnode); - pMgmt->pQnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c deleted file mode 100644 index b27a25680a193dfdffdecab08774ce7f33046e91..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ /dev/null @@ -1,445 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndSnode.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "dndWorker.h" - -typedef struct { - int32_t vgId; - int32_t refCount; - int32_t snVersion; - int8_t dropped; - char *path; - SSnode *pImpl; - STaosQueue *pSharedQ; - STaosQueue *pUniqueQ; -} SSnodeObj; - -static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg); - -static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); - -static SSnode *dndAcquireSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - SSnode *pSnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pSnode != NULL) { - refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); - pSnode = pMgmt->pSnode; - } else { - terrno = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - } - taosRUnLockLatch(&pMgmt->latch); - - if (pSnode != NULL) { - dTrace("acquire snode, refCount:%d", refCount); - } - return pSnode; -} - -static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) { - if (pSnode == NULL) return; - - SSnodeMgmt *pMgmt = &pDnode->smgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("release snode, refCount:%d", refCount); -} - -static int32_t dndReadSnodeFile(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - int32_t code = TSDB_CODE_DND_SNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_SNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_SNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_SNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); - goto PRASE_SNODE_OVER; - } - pMgmt->deployed = deployed->valueint; - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_SNODE_OVER; - } - pMgmt->dropped = dropped->valueint; - - code = 0; - dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); - -PRASE_SNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - terrno = code; - return code; -} - -static int32_t dndWriteSnodeFile(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - char file[PATH_MAX + 20]; - snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - - char realfile[PATH_MAX + 20]; - snprintf(realfile, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - - if (taosRenameFile(file, realfile) != 0) { - terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped); - return 0; -} - -static int32_t dndStartSnodeWorker(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); - for (int32_t i = 0; i < 2; i++) { - SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker)); - if (pUniqueWorker == NULL) { - return -1; - } - if (dndInitWorker(pDnode, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) { - dError("failed to start snode unique worker since %s", terrstr()); - return -1; - } - taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker); - } - if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", 4, 4, - dndProcessSnodeSharedQueue)) { - dError("failed to start snode shared worker since %s", terrstr()); - return -1; - } - - return 0; -} - -static void dndStopSnodeWorker(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - taosWLockLatch(&pMgmt->latch); - pMgmt->deployed = 0; - taosWUnLockLatch(&pMgmt->latch); - - while (pMgmt->refCount > 0) { - taosMsleep(10); - } - - for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { - SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i); - dndCleanupWorker(worker); - } - taosArrayDestroy(pMgmt->uniqueWorkers); -} - -static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { - pOption->pDnode = pDnode; - pOption->sendReqToDnodeFp = dndSendReqToDnode; - pOption->sendReqToMnodeFp = dndSendReqToMnode; - pOption->sendRedirectRspFp = dndSendRedirectRsp; - pOption->dnodeId = dndGetDnodeId(pDnode); - pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = tsVersion; -} - -static int32_t dndOpenSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - dndReleaseSnode(pDnode, pSnode); - terrno = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; - dError("failed to create snode since %s", terrstr()); - return -1; - } - - SSnodeOpt option = {0}; - dndBuildSnodeOption(pDnode, &option); - - pSnode = sndOpen(pDnode->dir.snode, &option); - if (pSnode == NULL) { - dError("failed to open snode since %s", terrstr()); - return -1; - } - - if (dndStartSnodeWorker(pDnode) != 0) { - dError("failed to start snode worker since %s", terrstr()); - sndClose(pSnode); - return -1; - } - - pMgmt->deployed = 1; - if (dndWriteSnodeFile(pDnode) != 0) { - pMgmt->deployed = 0; - dError("failed to write snode file since %s", terrstr()); - dndStopSnodeWorker(pDnode); - sndClose(pSnode); - return -1; - } - - taosWLockLatch(&pMgmt->latch); - pMgmt->pSnode = pSnode; - taosWUnLockLatch(&pMgmt->latch); - - dInfo("snode open successfully"); - return 0; -} - -static int32_t dndDropSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode == NULL) { - dError("failed to drop snode since %s", terrstr()); - return -1; - } - - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 1; - taosRUnLockLatch(&pMgmt->latch); - - if (dndWriteSnodeFile(pDnode) != 0) { - taosRLockLatch(&pMgmt->latch); - pMgmt->dropped = 0; - taosRUnLockLatch(&pMgmt->latch); - - dndReleaseSnode(pDnode, pSnode); - dError("failed to drop snode since %s", terrstr()); - return -1; - } - - dndReleaseSnode(pDnode, pSnode); - dndStopSnodeWorker(pDnode); - pMgmt->deployed = 0; - dndWriteSnodeFile(pDnode); - sndClose(pSnode); - pMgmt->pSnode = NULL; - sndDestroy(pDnode->dir.snode); - - return 0; -} - -int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateSnodeReq createReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION; - dError("failed to create snode since %s", terrstr()); - return -1; - } else { - return dndOpenSnode(pDnode); - } -} - -int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropSnodeReq dropReq = {0}; - if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION; - dError("failed to drop snode since %s", terrstr()); - return -1; - } else { - return dndDropSnode(pDnode); - } -} - -static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - for (int32_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - - sndProcessUMsg(pSnode, pMsg); - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - } - dndReleaseSnode(pDnode, pSnode); -} - -static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - code = sndProcessSMsg(pSnode, pMsg); - } - dndReleaseSnode(pDnode, pSnode); - -#if 0 - if (pMsg->msgType & 1u) { - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } -#endif - - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dndWriteSnodeMsgToRandomWorker(SDnode *pDnode, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - int32_t index = (pDnode->smgmt.uniqueWorkerInUse + 1) % taosArrayGetSize(pDnode->smgmt.uniqueWorkers); - SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, index); - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseSnode(pDnode, pSnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) { - int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; - - SSnode *pSnode = dndAcquireSnode(pDnode); - if (pSnode != NULL) { - code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)); - } - dndReleaseSnode(pDnode, pSnode); - - if (code != 0) { - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } -} - -void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - // judge from msg to write to unique queue - dndWriteSnodeMsgToRandomWorker(pDnode, pMsg); -} - -void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg); -} - -int32_t dndInitSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - taosInitRWLatch(&pMgmt->latch); - - if (dndReadSnodeFile(pDnode) != 0) { - return -1; - } - - if (pMgmt->dropped) { - dInfo("snode has been deployed and needs to be deleted"); - sndDestroy(pDnode->dir.snode); - return 0; - } - - if (!pMgmt->deployed) return 0; - - return dndOpenSnode(pDnode); -} - -void dndCleanupSnode(SDnode *pDnode) { - SSnodeMgmt *pMgmt = &pDnode->smgmt; - if (pMgmt->pSnode) { - dndStopSnodeWorker(pDnode); - sndClose(pMgmt->pSnode); - pMgmt->pSnode = NULL; - } -} diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c deleted file mode 100644 index b58b12bba2282026643ce79e655d096efb597c50..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/* this file is mainly responsible for the communication between DNODEs. Each - * dnode works as both server and client. Dnode may send status, grant, config - * messages to mnode, mnode may send create/alter/drop table/vnode messages - * to dnode. All theses messages are handled from here - */ - -#define _DEFAULT_SOURCE -#include "dndTransport.h" -#include "dndMgmt.h" -#include "dndMnode.h" -#include "dndVnodes.h" - -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" -#define INTERNAL_SECRET "_pwd" - -static void dndInitMsgFp(STransMgmt *pMgmt) { - // Requests handled by DNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_NETWORK_TEST)] = dndProcessMgmtMsg; - - // Requests handled by MNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONNECT)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_ACCT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_USER)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONFIG_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_QNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_QNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_SNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_SNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_BNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_BNODE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TABLE_META)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYSTABLE_RETRIEVE)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_TRANS)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH_RSP)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg; - - // Requests handled by VNODE - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH_RSP)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONSUME)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_QUERY)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONNECT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_DISCONNECT)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_RES_READY)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TASKS_STATUS)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CANCEL_TASK)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TASK)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_HEARTBEAT)] = dndProcessVnodeFetchMsg; -} - -static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode * pDnode = parent; - STransMgmt *pMgmt = &pDnode->tmgmt; - - tmsg_t msgType = pRsp->msgType; - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - if (pRsp == NULL || pRsp->pCont == NULL) return; - dTrace("RPC %p, rsp:%s ignored since dnode exiting, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - return; - } - - DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; - if (fp != NULL) { - dTrace("RPC %p, rsp:%s will be processed, code:0x%x app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF, - pRsp->ahandle); - (*fp)(pDnode, pRsp, pEpSet); - } else { - dError("RPC %p, rsp:%s not processed, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle); - rpcFreeCont(pRsp->pCont); - } -} - -static int32_t dndInitClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "D-C"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dndProcessResponse; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = INTERNAL_USER; - rpcInit.ckey = INTERNAL_CKEY; - rpcInit.spi = 1; - rpcInit.parent = pDnode; - - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - rpcInit.secret = pass; - - pMgmt->clientRpc = rpcOpen(&rpcInit); - if (pMgmt->clientRpc == NULL) { - dError("failed to init dnode rpc client"); - return -1; - } - - dDebug("dnode rpc client is initialized"); - return 0; -} - -static void dndCleanupClient(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->clientRpc) { - rpcClose(pMgmt->clientRpc); - pMgmt->clientRpc = NULL; - dDebug("dnode rpc client is closed"); - } -} - -static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode * pDnode = param; - STransMgmt *pMgmt = &pDnode->tmgmt; - - tmsg_t msgType = pReq->msgType; - if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("RPC %p, network test req will be processed, app:%p", pReq->handle, pReq->ahandle); - dndProcessStartupReq(pDnode, pReq); - return; - } - - if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("RPC %p, req:%s ignored since dnode exiting, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) { - dError("RPC %p, req:%s ignored since dnode not running, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } - - if (pReq->pCont == NULL) { - dTrace("RPC %p, req:%s not processed since its empty, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - return; - } - - DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; - if (fp != NULL) { - dTrace("RPC %p, req:%s will be processed, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - (*fp)(pDnode, pReq, pEpSet); - } else { - dError("RPC %p, req:%s not processed since no handle, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - } -} - -static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { - STransMgmt *pMgmt = &pDnode->tmgmt; - - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); -} - -static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - if (strcmp(user, INTERNAL_USER) == 0) { - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - return 0; - } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - return 0; - } else { - return -1; - } -} - -static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SDnode *pDnode = parent; - - if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - if (terrno != TSDB_CODE_APP_NOT_READY) { - dTrace("failed to get user auth from mnode since %s", terrstr()); - return -1; - } - - SAuthReq authReq = {0}; - tstrncpy(authReq.user, user, TSDB_USER_LEN); - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void * pReq = rpcMallocCont(contLen); - tSerializeSAuthReq(pReq, contLen, &authReq); - - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; - SRpcMsg rpcRsp = {0}; - dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); - dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - terrno = rpcRsp.code; - dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); - } else { - SAuthRsp authRsp = {0}; - tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); - memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); - memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); - *spi = authRsp.spi; - *encrypt = authRsp.encrypt; - dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, - authRsp.encrypt); - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - -static int32_t dndInitServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - dndInitMsgFp(pMgmt); - - int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); - if (numOfThreads < 1) { - numOfThreads = 1; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = pDnode->cfg.serverPort; - rpcInit.label = "D-S"; - rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = tsMaxShellConns; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dndRetrieveUserAuthInfo; - rpcInit.parent = pDnode; - - pMgmt->serverRpc = rpcOpen(&rpcInit); - if (pMgmt->serverRpc == NULL) { - dError("failed to init dnode rpc server"); - return -1; - } - - dDebug("dnode rpc server is initialized"); - return 0; -} - -static void dndCleanupServer(SDnode *pDnode) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->serverRpc) { - rpcClose(pMgmt->serverRpc); - pMgmt->serverRpc = NULL; - dDebug("dnode rpc server is closed"); - } -} - -int32_t dndInitTrans(SDnode *pDnode) { - if (dndInitClient(pDnode) != 0) { - return -1; - } - - if (dndInitServer(pDnode) != 0) { - return -1; - } - - dInfo("dnode-transport is initialized"); - return 0; -} - -void dndCleanupTrans(SDnode *pDnode) { - dInfo("dnode-transport start to clean up"); - dndCleanupServer(pDnode); - dndCleanupClient(pDnode); - dInfo("dnode-transport is cleaned up"); -} - -int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq) { - STransMgmt *pMgmt = &pDnode->tmgmt; - if (pMgmt->clientRpc == NULL) { - terrno = TSDB_CODE_DND_OFFLINE; - return -1; - } - - rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL); - return 0; -} - -int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pReq) { - SEpSet epSet = {0}; - dndGetMnodeEpSet(pDnode, &epSet); - return dndSendReqToDnode(pDnode, &epSet, pReq); -} diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c deleted file mode 100644 index d311e1e41703e70ab0846d92f6946c81648b651c..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ /dev/null @@ -1,1024 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndVnodes.h" -#include "dndMgmt.h" -#include "dndTransport.h" -#include "sync.h" - -typedef struct { - int32_t vgId; - int32_t vgVersion; - int8_t dropped; - uint64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; - char path[PATH_MAX + 20]; -} SWrapperCfg; - -typedef struct { - int32_t vgId; - int32_t refCount; - int32_t vgVersion; - int8_t dropped; - int8_t accessState; - uint64_t dbUid; - char *db; - char *path; - SVnode *pImpl; - STaosQueue *pWriteQ; - STaosQueue *pSyncQ; - STaosQueue *pApplyQ; - STaosQueue *pQueryQ; - STaosQueue *pFetchQ; -} SVnodeObj; - -typedef struct { - int32_t vnodeNum; - int32_t opened; - int32_t failed; - int32_t threadIndex; - pthread_t thread; - SDnode *pDnode; - SWrapperCfg *pCfgs; -} SVnodeThread; - -static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); - -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg); - -static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId); -static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl); -static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes); -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); -static int32_t dndWriteVnodesToFile(SDnode *pDnode); - -static int32_t dndOpenVnodes(SDnode *pDnode); -static void dndCloseVnodes(SDnode *pDnode); - -static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj *pVnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&pMgmt->latch); - taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); - if (pVnode == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - } else { - refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - } - taosRUnLockLatch(&pMgmt->latch); - - if (pVnode != NULL) { - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - } - - return pVnode; -} - -static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { - if (pVnode == NULL) return; - - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosRLockLatch(&pMgmt->latch); - int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - taosRUnLockLatch(&pMgmt->latch); - dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); -} - -static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); - if (pVnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pVnode->vgId = pCfg->vgId; - pVnode->refCount = 0; - pVnode->dropped = 0; - pVnode->accessState = TSDB_VN_ALL_ACCCESS; - pVnode->pImpl = pImpl; - pVnode->vgVersion = pCfg->vgVersion; - pVnode->dbUid = pCfg->dbUid; - pVnode->db = tstrdup(pCfg->db); - pVnode->path = tstrdup(pCfg->path); - - if (pVnode->path == NULL || pVnode->db == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (dndAllocVnodeQueue(pDnode, pVnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - taosWLockLatch(&pMgmt->latch); - int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); - taosWUnLockLatch(&pMgmt->latch); - - if (code != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } - return code; -} - -static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosWLockLatch(&pMgmt->latch); - taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); - taosWUnLockLatch(&pMgmt->latch); - - dndReleaseVnode(pDnode, pVnode); - while (pVnode->refCount > 0) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); - - dndFreeVnodeQueue(pDnode, pVnode); - vnodeClose(pVnode->pImpl); - pVnode->pImpl = NULL; - - dDebug("vgId:%d, vnode is closed", pVnode->vgId); - - if (pVnode->dropped) { - dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped); - vnodeDestroy(pVnode->path); - } - - free(pVnode->path); - free(pVnode->db); - free(pVnode); -} - -static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosRLockLatch(&pMgmt->latch); - - int32_t num = 0; - int32_t size = taosHashGetSize(pMgmt->hash); - SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); - - void *pIter = taosHashIterate(pMgmt->hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - SVnodeObj *pVnode = *ppVnode; - if (pVnode && num < size) { - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - pVnodes[num] = (*ppVnode); - num++; - pIter = taosHashIterate(pMgmt->hash, pIter); - } else { - taosHashCancelIterate(pMgmt->hash, pIter); - } - } - - taosRUnLockLatch(&pMgmt->latch); - *numOfVnodes = num; - - return pVnodes; -} - -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 30000; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - FILE *fp = NULL; - char file[PATH_MAX + 20] = {0}; - SWrapperCfg *pCfgs = NULL; - - snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - - // fp = fopen(file, "r"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_VNODE_OVER; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_VNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_VNODE_OVER; - } - - cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); - if (!vnodes || vnodes->type != cJSON_Array) { - dError("failed to read %s since vnodes not found", file); - goto PRASE_VNODE_OVER; - } - - int32_t vnodesNum = cJSON_GetArraySize(vnodes); - if (vnodesNum > 0) { - pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); - if (pCfgs == NULL) { - dError("failed to read %s since out of memory", file); - goto PRASE_VNODE_OVER; - } - - for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); - SWrapperCfg *pCfg = &pCfgs[i]; - - cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_Number) { - dError("failed to read %s since vgId not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->vgId = vgId->valueint; - snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId); - - cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->dropped = dropped->valueint; - - cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); - if (!vgVersion || vgVersion->type != cJSON_Number) { - dError("failed to read %s since vgVersion not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->vgVersion = vgVersion->valueint; - - cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); - if (!dbUid || dbUid->type != cJSON_String) { - dError("failed to read %s since dbUid not found", file); - goto PRASE_VNODE_OVER; - } - pCfg->dbUid = atoll(dbUid->valuestring); - - cJSON *db = cJSON_GetObjectItem(vnode, "db"); - if (!db || db->type != cJSON_String) { - dError("failed to read %s since db not found", file); - goto PRASE_VNODE_OVER; - } - tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN); - } - - *ppCfgs = pCfgs; - } - - *numOfVnodes = vnodesNum; - code = 0; - dInfo("succcessed to read file %s", file); - -PRASE_VNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (pFile != NULL) taosCloseFile(&pFile); - - return code; -} - -static int32_t dndWriteVnodesToFile(SDnode *pDnode) { - char file[PATH_MAX + 20] = {0}; - char realfile[PATH_MAX + 20] = {0}; - snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); - snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - - // FILE *fp = fopen(file, "w"); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); - - int32_t len = 0; - int32_t maxLen = 65536; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - len += snprintf(content + len, maxLen - len, " {\n"); - len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); - len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); - len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); - len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); - if (i < numOfVnodes - 1) { - len += snprintf(content + len, maxLen - len, " },\n"); - } else { - len += snprintf(content + len, maxLen - len, " }\n"); - } - } - len += snprintf(content + len, maxLen - len, " ]\n"); - len += snprintf(content + len, maxLen - len, "}\n"); - - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - free(content); - terrno = 0; - - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - dndReleaseVnode(pDnode, pVnode); - } - - if (pVnodes != NULL) { - free(pVnodes); - } - - dDebug("successed to write %s", realfile); - return taosRenameFile(file, realfile); -} - -static void *dnodeOpenVnodeFunc(void *param) { - SVnodeThread *pThread = param; - SDnode *pDnode = pThread->pDnode; - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); - setThreadName("open-vnodes"); - - for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - SWrapperCfg *pCfg = &pThread->pCfgs[v]; - - char stepDesc[TSDB_STEP_DESC_LEN] = {0}; - snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, - pMgmt->stat.openVnodes, pMgmt->stat.totalVnodes); - dndReportStartup(pDnode, "open-vnodes", stepDesc); - - SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; - SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); - if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); - pThread->failed++; - } else { - dndOpenVnode(pDnode, pCfg, pImpl); - dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); - pThread->opened++; - } - - atomic_add_fetch_32(&pMgmt->stat.openVnodes, 1); - } - - dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); - return NULL; -} - -static int32_t dndOpenVnodes(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosInitRWLatch(&pMgmt->latch); - - pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - if (pMgmt->hash == NULL) { - dError("failed to init vnode hash"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - SWrapperCfg *pCfgs = NULL; - int32_t numOfVnodes = 0; - if (dndGetVnodesFromFile(pDnode, &pCfgs, &numOfVnodes) != 0) { - dInfo("failed to get vnode list from disk since %s", terrstr()); - return -1; - } - - pMgmt->stat.totalVnodes = numOfVnodes; - - int32_t threadNum = tsNumOfCores; -#if 1 - threadNum = 1; -#endif - - int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - - SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); - for (int32_t t = 0; t < threadNum; ++t) { - threads[t].threadIndex = t; - threads[t].pDnode = pDnode; - threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); - } - - for (int32_t v = 0; v < numOfVnodes; ++v) { - int32_t t = v % threadNum; - SVnodeThread *pThread = &threads[t]; - pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; - } - - dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); - - for (int32_t t = 0; t < threadNum; ++t) { - SVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum == 0) continue; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnodeFunc, pThread) != 0) { - dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - } - - for (int32_t t = 0; t < threadNum; ++t) { - SVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { - pthread_join(pThread->thread, NULL); - } - free(pThread->pCfgs); - } - free(threads); - free(pCfgs); - - if (pMgmt->stat.openVnodes != pMgmt->stat.totalVnodes) { - dError("there are total vnodes:%d, opened:%d", pMgmt->stat.totalVnodes, pMgmt->stat.openVnodes); - return -1; - } else { - dInfo("total vnodes:%d open successfully", pMgmt->stat.totalVnodes); - return 0; - } -} - -static void dndCloseVnodes(SDnode *pDnode) { - dInfo("start to close all vnodes"); - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); - - for (int32_t i = 0; i < numOfVnodes; ++i) { - dndCloseVnode(pDnode, pVnodes[i]); - } - - if (pVnodes != NULL) { - free(pVnodes); - } - - if (pMgmt->hash != NULL) { - taosHashCleanup(pMgmt->hash); - pMgmt->hash = NULL; - } - - dInfo("total vnodes:%d are all closed", numOfVnodes); -} - -static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { - pCfg->vgId = pCreate->vgId; - pCfg->wsize = pCreate->cacheBlockSize; - pCfg->ssize = pCreate->cacheBlockSize; - pCfg->lsize = pCreate->cacheBlockSize; - pCfg->isHeapAllocator = true; - pCfg->ttl = 4; - pCfg->keep = pCreate->daysToKeep0; - pCfg->streamMode = pCreate->streamMode; - pCfg->isWeak = true; - pCfg->tsdbCfg.keep = pCreate->daysToKeep0; - pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; - pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; - pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; - pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; - pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; - pCfg->walCfg.level = pCreate->walLevel; - pCfg->walCfg.retentionPeriod = 10; - pCfg->walCfg.retentionSize = 128; - pCfg->walCfg.rollPeriod = 128; - pCfg->walCfg.segSize = 128; - pCfg->walCfg.vgId = pCreate->vgId; - pCfg->hashBegin = pCreate->hashBegin; - pCfg->hashEnd = pCreate->hashEnd; - pCfg->hashMethod = pCreate->hashMethod; -} - -static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { - memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN); - pCfg->dbUid = pCreate->dbUid; - pCfg->dropped = 0; - snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId); - pCfg->vgId = pCreate->vgId; - pCfg->vgVersion = pCreate->vgVersion; -} - -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SCreateVnodeReq createReq = {0}; - if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - dDebug("vgId:%d, create vnode req is received", createReq.vgId); - - SVnodeCfg vnodeCfg = {0}; - dndGenerateVnodeCfg(&createReq, &vnodeCfg); - - SWrapperCfg wrapperCfg = {0}; - dndGenerateWrapperCfg(pDnode, &createReq, &wrapperCfg); - - if (createReq.dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; - dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); - return -1; - } - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, createReq.vgId); - if (pVnode != NULL) { - dDebug("vgId:%d, already exist", createReq.vgId); - dndReleaseVnode(pDnode, pVnode); - terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; - return -1; - } - - vnodeCfg.pDnode = pDnode; - vnodeCfg.pTfs = pDnode->pTfs; - vnodeCfg.dbId = wrapperCfg.dbUid; - SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); - if (pImpl == NULL) { - dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); - return -1; - } - - int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); - if (code != 0) { - dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); - vnodeClose(pImpl); - vnodeDestroy(wrapperCfg.path); - terrno = code; - return code; - } - - code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(wrapperCfg.path); - terrno = code; - return code; - } - - return 0; -} - -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SAlterVnodeReq alterReq = {0}; - if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - dDebug("vgId:%d, alter vnode req is received", alterReq.vgId); - - SVnodeCfg vnodeCfg = {0}; - dndGenerateVnodeCfg(&alterReq, &vnodeCfg); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, alterReq.vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); - return -1; - } - - if (alterReq.vgVersion == pVnode->vgVersion) { - dndReleaseVnode(pDnode, pVnode); - dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId); - return 0; - } - - if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { - dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - int32_t oldVersion = pVnode->vgVersion; - pVnode->vgVersion = alterReq.vgVersion; - int32_t code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - pVnode->vgVersion = oldVersion; - } - - dndReleaseVnode(pDnode, pVnode); - return code; -} - -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDropVnodeReq dropReq = {0}; - if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - int32_t vgId = dropReq.vgId; - dDebug("vgId:%d, drop vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; - return -1; - } - - pVnode->dropped = 1; - if (dndWriteVnodesToFile(pDnode) != 0) { - pVnode->dropped = 0; - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndCloseVnode(pDnode, pVnode); - dndWriteVnodesToFile(pDnode); - - return 0; -} - -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SSyncVnodeReq syncReq = {0}; - tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq); - - int32_t vgId = syncReq.vgId; - dDebug("vgId:%d, sync vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); - return -1; - } - - if (vnodeSync(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndReleaseVnode(pDnode, pVnode); - return 0; -} - -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SCompactVnodeReq compatcReq = {0}; - tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq); - - int32_t vgId = compatcReq.vgId; - dDebug("vgId:%d, compact vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); - return -1; - } - - if (vnodeCompact(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); - return -1; - } - - dndReleaseVnode(pDnode, pVnode); - return 0; -} - -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessQueryMsg(pVnode->pImpl, pMsg); } - -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessFetchMsg(pVnode->pImpl, pMsg); } - -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - void *ptr = taosArrayPush(pArray, &pMsg); - assert(ptr != NULL); - } - - vnodeProcessWMsgs(pVnode->pImpl, pArray); - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pRsp = NULL; - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - int32_t code = vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); - if (pRsp != NULL) { - pRsp->ahandle = pMsg->ahandle; - rpcSendResponse(pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; - rpcSendResponse(&rpcRsp); - } - } - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - - taosArrayDestroy(pArray); -} - -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SRpcMsg *pMsg = NULL; - - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pMsg); - - // todo - SRpcMsg *pRsp = NULL; - (void)vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); - } -} - -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { - SRpcMsg *pMsg = NULL; - - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pMsg); - - // todo - SRpcMsg *pRsp = NULL; - (void)vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp); - } -} - -static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg, bool sendRsp) { - int32_t code = 0; - - if (pQueue == NULL) { - code = TSDB_CODE_MSG_NOT_PROCESSED; - } else { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - *pMsg = *pRpcMsg; - if (taosWriteQitem(pQueue, pMsg) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } - } - - if (code != TSDB_CODE_SUCCESS && sendRsp) { - if (pRpcMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pRpcMsg->pCont); - } - - return code; -} - -static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - pHead->contLen = htonl(pHead->contLen); - pHead->vgId = htonl(pHead->vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); - if (pVnode == NULL) { - dError("vgId:%d, failed to acquire vnode while process req", pHead->vgId); - if (pMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; - rpcSendResponse(&rsp); - } - rpcFreeCont(pMsg->pCont); - } - - return pVnode; -} - -void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); - if (pVnode != NULL) { - (void)dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, true); - dndReleaseVnode(pDnode, pVnode); - } -} - -int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pMsg) { - SMsgHead *pHead = pMsg->pCont; - // pHead->vgId = htonl(pHead->vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); - if (pVnode == NULL) return -1; - - int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, false); - dndReleaseVnode(pDnode, pVnode); - return code; -} - -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) return -1; - - int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg); - dndReleaseVnode(pDnode, pVnode); - return code; -} - -static int32_t dndInitVnodeWorkers(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - int32_t maxFetchThreads = 4; - int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); - int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); - int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); - int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); - - SQWorkerPool *pQPool = &pMgmt->queryPool; - pQPool->name = "vnode-query"; - pQPool->min = minQueryThreads; - pQPool->max = maxQueryThreads; - if (tQWorkerInit(pQPool) != 0) return -1; - - SFWorkerPool *pFPool = &pMgmt->fetchPool; - pFPool->name = "vnode-fetch"; - pFPool->min = minFetchThreads; - pFPool->max = maxFetchThreads; - if (tFWorkerInit(pFPool) != 0) return -1; - - SWWorkerPool *pWPool = &pMgmt->writePool; - pWPool->name = "vnode-write"; - pWPool->max = maxWriteThreads; - if (tWWorkerInit(pWPool) != 0) return -1; - - pWPool = &pMgmt->syncPool; - pWPool->name = "vnode-sync"; - pWPool->max = maxSyncThreads; - if (tWWorkerInit(pWPool) != 0) return -1; - - dDebug("vnode workers is initialized"); - return 0; -} - -static void dndCleanupVnodeWorkers(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tFWorkerCleanup(&pMgmt->fetchPool); - tQWorkerCleanup(&pMgmt->queryPool); - tWWorkerCleanup(&pMgmt->writePool); - tWWorkerCleanup(&pMgmt->syncPool); - dDebug("vnode workers is closed"); -} - -static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - - pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeWriteQueue); - pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeApplyQueue); - pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)dndProcessVnodeSyncQueue); - pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)dndProcessVnodeFetchQueue); - pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)dndProcessVnodeQueryQueue); - - if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || - pVnode->pQueryQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); - tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); - tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); - tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); - pVnode->pWriteQ = NULL; - pVnode->pApplyQ = NULL; - pVnode->pSyncQ = NULL; - pVnode->pFetchQ = NULL; - pVnode->pQueryQ = NULL; -} - -int32_t dndInitVnodes(SDnode *pDnode) { - dInfo("dnode-vnodes start to init"); - - if (dndInitVnodeWorkers(pDnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - dError("failed to init vnode workers since %s", terrstr()); - return -1; - } - - if (dndOpenVnodes(pDnode) != 0) { - dError("failed to open vnodes since %s", terrstr()); - return -1; - } - - dInfo("dnode-vnodes is initialized"); - return 0; -} - -void dndCleanupVnodes(SDnode *pDnode) { - dInfo("dnode-vnodes start to clean up"); - dndCloseVnodes(pDnode); - dndCleanupVnodeWorkers(pDnode); - dInfo("dnode-vnodes is cleaned up"); -} - -void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodesStat *pStat = &pMgmt->stat; - int32_t totalVnodes = 0; - int32_t masterNum = 0; - int64_t numOfSelectReqs = 0; - int64_t numOfInsertReqs = 0; - int64_t numOfInsertSuccessReqs = 0; - int64_t numOfBatchInsertReqs = 0; - int64_t numOfBatchInsertSuccessReqs = 0; - - taosRLockLatch(&pMgmt->latch); - - void *pIter = taosHashIterate(pMgmt->hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - if (ppVnode == NULL || *ppVnode == NULL) continue; - - SVnodeObj *pVnode = *ppVnode; - SVnodeLoad vload = {0}; - vnodeGetLoad(pVnode->pImpl, &vload); - taosArrayPush(pLoads, &vload); - - numOfSelectReqs += vload.numOfSelectReqs; - numOfInsertReqs += vload.numOfInsertReqs; - numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs; - numOfBatchInsertReqs += vload.numOfBatchInsertReqs; - numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs; - totalVnodes++; - if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++; - - pIter = taosHashIterate(pMgmt->hash, pIter); - } - - taosRUnLockLatch(&pMgmt->latch); - - pStat->totalVnodes = totalVnodes; - pStat->masterNum = masterNum; - pStat->numOfSelectReqs = numOfSelectReqs; - pStat->numOfInsertReqs = numOfInsertReqs; - pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs; - pStat->numOfBatchInsertReqs = numOfBatchInsertReqs; - pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; -} diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/main/CMakeLists.txt similarity index 62% rename from source/dnode/mgmt/daemon/CMakeLists.txt rename to source/dnode/mgmt/main/CMakeLists.txt index 3238bbf3f07279e590350d060bb1a8d4ad47c7ef..baa486b91fe3d039a127b47c76576ded80d3c1b9 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/main/CMakeLists.txt @@ -1,5 +1,5 @@ -aux_source_directory(src DAEMON_SRC) -add_executable(taosd ${DAEMON_SRC}) +aux_source_directory(src EXEC_SRC) +add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndMain.h similarity index 58% rename from source/dnode/mgmt/impl/inc/dndInt.h rename to source/dnode/mgmt/main/inc/dndMain.h index a8530037daabecd6c2fad43f8ad7f680315bd6e4..1958d628a0ad2ba2a2c28c4a835ee6a5ff13f37e 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndMain.h @@ -1,3 +1,4 @@ + /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -13,39 +14,20 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_INT_H_ -#define _TD_DND_INT_H_ +#ifndef _TD_DND_MAIN_H_ +#define _TD_DND_MAIN_H_ -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" +#include "dnode.h" -#include "cJSON.h" -#include "monitor.h" -#include "tcache.h" -#include "tcrc32c.h" -#include "tdatablock.h" +#include "taoserror.h" +#include "tconfig.h" #include "tglobal.h" -#include "thash.h" -#include "tlockfree.h" #include "tlog.h" -#include "tmsg.h" -#include "tqueue.h" -#include "trpc.h" -#include "tthread.h" -#include "ttime.h" -#include "tworker.h" +#include "version.h" -#include "dnode.h" - -#include "bnode.h" -#include "mnode.h" -#include "qnode.h" -#include "snode.h" -#include "vnode.h" -#include "tfs.h" +#ifdef __cplusplus +extern "C" { +#endif #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} @@ -54,20 +36,13 @@ extern "C" { #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} -typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; -typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; -typedef enum { DND_ENV_INIT = 0, DND_ENV_READY = 1, DND_ENV_CLEANUP = 2 } EEnvStat; -typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); - -EStat dndGetStat(SDnode *pDnode); -void dndSetStat(SDnode *pDnode, EStat stat); -const char *dndStatStr(EStat stat); - -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); +void dndDumpCfg(); +void dndPrintVersion(); +void dndGenerateGrant(); +SDnodeOpt dndGetOpt(); #ifdef __cplusplus } #endif -#endif /*_TD_DND_INT_H_*/ +#endif /*_TD_DND_MAIN_H_*/ diff --git a/source/dnode/mgmt/main/src/dndMain.c b/source/dnode/mgmt/main/src/dndMain.c new file mode 100644 index 0000000000000000000000000000000000000000..3aff3446dac278ff38636c6dfca633b25c0a2afd --- /dev/null +++ b/source/dnode/mgmt/main/src/dndMain.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndMain.h" + +static struct { + bool dumpConfig; + bool generateGrant; + bool printAuth; + bool printVersion; + char envFile[PATH_MAX]; + char apolloUrl[PATH_MAX]; + SDnode *pDnode; +} global = {0}; + +static void dndSigintHandle(int signum, void *info, void *ctx) { + dInfo("signal:%d is received", signum); + SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode); + if (pDnode != NULL) { + dndHandleEvent(pDnode, DND_EVENT_STOP); + } +} + +static void dndSetSignalHandle() { + taosSetSignal(SIGTERM, dndSigintHandle); + taosSetSignal(SIGHUP, dndSigintHandle); + taosSetSignal(SIGINT, dndSigintHandle); + taosSetSignal(SIGABRT, dndSigintHandle); + taosSetSignal(SIGBREAK, dndSigintHandle); +} + +static int32_t dndParseOption(int32_t argc, char const *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-C") == 0) { + global.dumpConfig = true; + } else if (strcmp(argv[i], "-k") == 0) { + global.generateGrant = true; + } else if (strcmp(argv[i], "-V") == 0) { + global.printVersion = true; + } else { + } + } + + return 0; +} + +static int32_t dndRunDnode() { + if (dndInit() != 0) { + dInfo("failed to initialize dnode environment since %s", terrstr()); + return -1; + } + + SDnodeOpt option = dndGetOpt(); + + SDnode *pDnode = dndCreate(&option); + if (pDnode == NULL) { + dError("failed to to create dnode object since %s", terrstr()); + return -1; + } else { + global.pDnode = pDnode; + dndSetSignalHandle(); + } + + dInfo("start the TDengine service"); + int32_t code = dndRun(pDnode); + dInfo("start shutting down the TDengine service"); + + global.pDnode = NULL; + dndClose(pDnode); + dndCleanup(); + taosCloseLog(); + taosCleanupCfg(); + return code; +} + +int main(int argc, char const *argv[]) { + if (!taosCheckSystemIsSmallEnd()) { + dError("failed to start TDengine since on non-small-end machines"); + return -1; + } + + if (dndParseOption(argc, argv) != 0) { + return -1; + } + + if (global.generateGrant) { + dndGenerateGrant(); + return 0; + } + + if (global.printVersion) { + dndPrintVersion(); + return 0; + } + + if (taosCreateLog("taosdlog", 1, configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) { + dError("failed to start TDengine since read log config error"); + return -1; + } + + if (taosInitCfg(configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) { + dError("failed to start TDengine since read config error"); + return -1; + } + + if (global.dumpConfig) { + dndDumpCfg(); + taosCleanupCfg(); + taosCloseLog(); + return 0; + } + + return dndRunDnode(); +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/main/src/dndUtil.c similarity index 55% rename from source/dnode/mgmt/daemon/src/dmnUtil.c rename to source/dnode/mgmt/main/src/dndUtil.c index d58a5968c0726ae99fd23bb5116bfa3269da1dd7..e07ef68c774fb70f789cd00d43d37a0a21b504f9 100644 --- a/source/dnode/mgmt/daemon/src/dmnUtil.c +++ b/source/dnode/mgmt/main/src/dndUtil.c @@ -14,15 +14,15 @@ */ #define _DEFAULT_SOURCE -#include "dmnInt.h" +#include "dndMain.h" -void dmnGenerateGrant() { +void dndGenerateGrant() { #if 0 grantParseParameter(); #endif } -void dmnPrintVersion() { +void dndPrintVersion() { #ifdef TD_ENTERPRISE char *releaseName = "enterprise"; #else @@ -32,3 +32,24 @@ void dmnPrintVersion() { printf("gitinfo: %s\n", gitinfo); printf("builuInfo: %s\n", buildinfo); } + +void dndDumpCfg() { + SConfig *pCfg = taosGetCfg(); + cfgDumpCfg(pCfg, 0, 1); +} + +SDnodeOpt dndGetOpt() { + SConfig *pCfg = taosGetCfg(); + SDnodeOpt option = {0}; + + option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir)); + tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp)); + tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp)); + option.serverPort = tsServerPort; + tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn)); + snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort); + option.pDisks = tsDiskCfg; + option.numOfDisks = tsDiskCfgNum; + return option; +} diff --git a/source/dnode/mgmt/mnode/inc/mm.h b/source/dnode/mgmt/mnode/inc/mm.h new file mode 100644 index 0000000000000000000000000000000000000000..6ed6c42d93c7bceacba6c386158abaef38b74baa --- /dev/null +++ b/source/dnode/mgmt/mnode/inc/mm.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_MNODE_H_ +#define _TD_DND_MNODE_H_ + +#include "dnd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void mmGetMgmtFp(SMgmtWrapper *pMgmt); + +int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mnode/inc/mmInt.h b/source/dnode/mgmt/mnode/inc/mmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..1751131764ec1b574a4eb4335ea53c4d61efbed1 --- /dev/null +++ b/source/dnode/mgmt/mnode/inc/mmInt.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_MNODE_INT_H_ +#define _TD_DND_MNODE_INT_H_ + +#include "mm.h" +#include "mnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SMnodeMgmt { + SMnode *pMnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SDnodeWorker readWorker; + SDnodeWorker writeWorker; + SDnodeWorker syncWorker; + SReplica replicas[TSDB_MAX_REPLICA]; + int8_t replica; + int8_t selfIndex; +} SMnodeMgmt; + +// mmFile.c +int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed); +int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed); + +// mmInt.c +int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq); +int32_t mmDrop(SMgmtWrapper *pWrapper); +int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq); + +// mmMsg.c +void mmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); + +// mmWorker.c +int32_t mmStartWorker(SMnodeMgmt *pMgmt); +void mmStopWorker(SMnodeMgmt *pMgmt); +int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg); +int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_MNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mnode/src/mmFile.c b/source/dnode/mgmt/mnode/src/mmFile.c new file mode 100644 index 0000000000000000000000000000000000000000..757b39747390076536279eb5311227a265e9c397 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmFile.c @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_MNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_MNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_MNODE_OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_Number) { + dError("failed to read %s since deployed not found", file); + goto PRASE_MNODE_OVER; + } + *pDeployed = deployed->valueint; + + cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); + if (!mnodes || mnodes->type != cJSON_Array) { + dError("failed to read %s since nodes not found", file); + goto PRASE_MNODE_OVER; + } + + pMgmt->replica = cJSON_GetArraySize(mnodes); + if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { + dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); + goto PRASE_MNODE_OVER; + } + + for (int32_t i = 0; i < pMgmt->replica; ++i) { + cJSON *node = cJSON_GetArrayItem(mnodes, i); + if (node == NULL) break; + + SReplica *pReplica = &pMgmt->replicas[i]; + + cJSON *id = cJSON_GetObjectItem(node, "id"); + if (!id || id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); + goto PRASE_MNODE_OVER; + } + pReplica->id = id->valueint; + + cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); + if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto PRASE_MNODE_OVER; + } + tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *port = cJSON_GetObjectItem(node, "port"); + if (!port || port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto PRASE_MNODE_OVER; + } + pReplica->port = port->valueint; + } + + code = 0; + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + +PRASE_MNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { + char file[PATH_MAX]; + snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno);; + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 4096; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", deployed); + len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); + for (int32_t i = 0; i < pMgmt->replica; ++i) { + SReplica *pReplica = &pMgmt->replicas[i]; + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); + if (i < pMgmt->replica - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + free(content); + + char realfile[PATH_MAX]; + snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + + if (taosRenameFile(file, realfile) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno);; + dError("failed to rename %s since %s", file, terrstr()); + return -1; + } + + dInfo("successed to write %s, deployed:%d", realfile, deployed); + return 0; +} diff --git a/source/dnode/mgmt/mnode/src/mmInt.c b/source/dnode/mgmt/mnode/src/mmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..a3c2d40510c47640ff5e54dc46270545c9b4c341 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmInt.c @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +static bool mmDeployRequired(SDnode *pDnode) { + if (pDnode->dnodeId > 0) return false; + if (pDnode->clusterId > 0) return false; + if (strcmp(pDnode->localEp, pDnode->firstEp) != 0) return false; + return true; +} + +static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) { + SMnodeMgmt mgmt = {0}; + mgmt.path = pWrapper->path; + if (mmReadFile(&mgmt, required) != 0) { + return -1; + } + + if (!(*required)) { + *required = mmDeployRequired(pWrapper->pDnode); + } + + return 0; +} + +static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->pWrapper = pMgmt->pWrapper; + pOption->putToWriteQFp = mmPutMsgToWriteQueue; + pOption->putToReadQFp = mmPutMsgToReadQueue; + pOption->sendReqFp = dndSendReqToDnode; + pOption->sendMnodeReqFp = dndSendReqToMnode; + pOption->sendRspFp = dndSendRsp; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; +} + +static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + + mmInitOption(pMgmt, pOption); + pOption->replica = 1; + pOption->selfIndex = 0; + SReplica *pReplica = &pOption->replicas[0]; + pReplica->id = 1; + pReplica->port = pDnode->serverPort; + tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN); + + pMgmt->selfIndex = pOption->selfIndex; + pMgmt->replica = pOption->replica; + memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); +} + +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { + mmInitOption(pMgmt, pOption); + pOption->selfIndex = pMgmt->selfIndex; + pOption->replica = pMgmt->replica; + memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); +} + +static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) { + mmInitOption(pMgmt, pOption); + + pOption->replica = pCreate->replica; + pOption->selfIndex = -1; + for (int32_t i = 0; i < pCreate->replica; ++i) { + SReplica *pReplica = &pOption->replicas[i]; + pReplica->id = pCreate->replicas[i].id; + pReplica->port = pCreate->replicas[i].port; + memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); + if (pReplica->id == pOption->dnodeId) { + pOption->selfIndex = i; + } + } + + if (pOption->selfIndex == -1) { + dError("failed to build mnode options since %s", terrstr()); + return -1; + } + + pMgmt->selfIndex = pOption->selfIndex; + pMgmt->replica = pOption->replica; + memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); + return 0; +} + +static int32_t mmOpenImp(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq) { + SMnodeOpt option = {0}; + if (pReq != NULL) { + if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) { + return -1; + } + } else { + bool deployed = false; + if (mmReadFile(pMgmt, &deployed) != 0) { + dError("failed to read file since %s", terrstr()); + return -1; + } + + if (!deployed) { + dInfo("mnode start to deploy"); + mmBuildOptionForDeploy(pMgmt, &option); + } else { + dInfo("mnode start to open"); + mmBuildOptionForOpen(pMgmt, &option); + } + } + + pMgmt->pMnode = mndOpen(pMgmt->path, &option); + if (pMgmt->pMnode == NULL) { + dError("failed to open mnode since %s", terrstr()); + return -1; + } + + if (mmStartWorker(pMgmt) != 0) { + dError("failed to start mnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (mmWriteFile(pMgmt, deployed) != 0) { + dError("failed to write mnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void mmCloseImp(SMnodeMgmt *pMgmt) { + if (pMgmt->pMnode != NULL) { + mmStopWorker(pMgmt); + mndClose(pMgmt->pMnode); + pMgmt->pMnode = NULL; + } +} + +int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq) { + SMnodeOpt option = {0}; + if (pReq != NULL) { + if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) { + return -1; + } + } + + return mndAlter(pMgmt->pMnode, &option); +} + +int32_t mmDrop(SMgmtWrapper *pWrapper) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("mnode-mgmt start to drop"); + bool deployed = false; + if (mmWriteFile(pMgmt, deployed) != 0) { + dError("failed to drop mnode since %s", terrstr()); + return -1; + } + + mmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("mnode-mgmt is dropped"); + return 0; +} + +static void mmClose(SMgmtWrapper *pWrapper) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("mnode-mgmt start to cleanup"); + mmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("mnode-mgmt is cleaned up"); +} + +int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) { + dInfo("mnode-mgmt start to init"); + if (walInit() != 0) { + dError("failed to init wal since %s", terrstr()); + return -1; + } + + SMnodeMgmt *pMgmt = calloc(1, sizeof(SMnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = mmOpenImp(pMgmt, pReq); + if (code != 0) { + dError("failed to init mnode-mgmt since %s", terrstr()); + mmClose(pWrapper); + } else { + dInfo("mnode-mgmt is initialized"); + } + + return code; +} + +static int32_t mmOpen(SMgmtWrapper *pWrapper) { + return mmOpenFromMsg(pWrapper, NULL); +} + +static int32_t mmStart(SMgmtWrapper *pWrapper) { + dDebug("mnode mgmt start to run"); + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mndStart(pMgmt->pMnode); +} + +void mmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = mmOpen; + mgmtFp.closeFp = mmClose; + mgmtFp.startFp = mmStart; + mgmtFp.createMsgFp = mmProcessCreateReq; + mgmtFp.dropMsgFp = mmProcessDropReq; + mgmtFp.requiredFp = mmRequire; + + mmInitMsgHandles(pWrapper); + pWrapper->name = "mnode"; + pWrapper->fp = mgmtFp; +} + +int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + + int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey); + dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); + return code; +} + +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); +} \ No newline at end of file diff --git a/source/dnode/mgmt/mnode/src/mmMsg.c b/source/dnode/mgmt/mnode/src/mmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..41fd23e3c7c2c3ccae488f296daf8f5cb1d6ccaa --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmMsg.c @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateMnodeReq createReq = {0}; + if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.replica <= 1 || createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create mnode since %s", terrstr()); + return -1; + } else { + return mmOpenFromMsg(pWrapper, &createReq); + } +} + +int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropMnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop mnode since %s", terrstr()); + return -1; + } else { + return mmDrop(pWrapper); + } +} + +int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnode *pDnode = pMgmt->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDAlterMnodeReq alterReq = {0}; + if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (alterReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to alter mnode since %s", terrstr()); + return -1; + } else { + return mmAlter(pMgmt, &alterReq); + } +} + +void mmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by DNODE + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + + // Requests handled by MNODE + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg); + + // Requests handled by VNODE + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); +} diff --git a/source/dnode/mgmt/mnode/src/mmWorker.c b/source/dnode/mgmt/mnode/src/mmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..4fa2f60b093bf882e75c95d75bcc47bce2234652 --- /dev/null +++ b/source/dnode/mgmt/mnode/src/mmWorker.c @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mmInt.h" + +static void mmProcessQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in mnode queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + if (pMsg->rpcMsg.msgType != TDMT_DND_ALTER_MNODE) { + pMsg->pNode = pMgmt->pMnode; + code = mndProcessMsg(pMsg); + } else { + code = mmProcessAlterReq(pMgmt, pMsg); + } + + if (pRpc->msgType & 1U) { + if (pRpc->handle == NULL) return; + if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0) code = terrno; + SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; + dndSendRsp(pMgmt->pWrapper, &rsp); + } + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + +int32_t mmStartWorker(SMnodeMgmt *pMgmt) { + if (dndInitWorker(pMgmt, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, mmProcessQueue) != 0) { + dError("failed to start mnode read worker since %s", terrstr()); + return -1; + } + + if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, mmProcessQueue) != 0) { + dError("failed to start mnode write worker since %s", terrstr()); + return -1; + } + + if (dndInitWorker(pMgmt, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, mmProcessQueue) != 0) { + dError("failed to start mnode sync worker since %s", terrstr()); + return -1; + } + + return 0; +} + +void mmStopWorker(SMnodeMgmt *pMgmt) { + dndCleanupWorker(&pMgmt->readWorker); + dndCleanupWorker(&pMgmt->writeWorker); + dndCleanupWorker(&pMgmt->syncWorker); +} + +static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SNodeMsg *pMsg) { + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); +} + +int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); +} + +int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); +} + +static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) { + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg == NULL) { + return -1; + } + + dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + pMsg->rpcMsg = *pRpc; + + int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); + if (code != 0) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + return code; +} + +int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->writeWorker, pRpc); +} + +int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc); +} diff --git a/source/dnode/mgmt/impl/inc/dndTransport.h b/source/dnode/mgmt/qnode/inc/qm.h similarity index 68% rename from source/dnode/mgmt/impl/inc/dndTransport.h rename to source/dnode/mgmt/qnode/inc/qm.h index e0ea21cba8e16f41bae17c6e5dccd0bb8558480e..e28ea3e9483f92c5405e6cf910330de973885e8c 100644 --- a/source/dnode/mgmt/impl/inc/dndTransport.h +++ b/source/dnode/mgmt/qnode/inc/qm.h @@ -13,21 +13,19 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_TRANSPORT_H_ -#define _TD_DND_TRANSPORT_H_ +#ifndef _TD_DND_QNODE_H_ +#define _TD_DND_QNODE_H_ + +#include "dnd.h" #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitTrans(SDnode *pDnode); -void dndCleanupTrans(SDnode *pDnode); -int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pRpcMsg); +void qmGetMgmtFp(SMgmtWrapper *pMgmt); #ifdef __cplusplus } #endif -#endif /*_TD_DND_TRANSPORT_H_*/ +#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qnode/inc/qmInt.h b/source/dnode/mgmt/qnode/inc/qmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..2629486d6034127f2a8689b0de29e52871377419 --- /dev/null +++ b/source/dnode/mgmt/qnode/inc/qmInt.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_QNODE_INT_H_ +#define _TD_DND_QNODE_INT_H_ + +#include "qm.h" +#include "qnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SQnodeMgmt { + SQnode *pQnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SDnodeWorker queryWorker; + SDnodeWorker fetchWorker; +} SQnodeMgmt; + +// qmInt.c +int32_t qmOpen(SMgmtWrapper *pWrapper); +int32_t qmDrop(SMgmtWrapper *pWrapper); + +// qmMsg.c +void qmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// qmWorker.c +int32_t qmStartWorker(SQnodeMgmt *pMgmt); +void qmStopWorker(SQnodeMgmt *pMgmt); +int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_QNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qnode/src/qmInt.c b/source/dnode/mgmt/qnode/src/qmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..78ef238ab89b95398eb430be8f4cbb26f3b3003d --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmInt.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->pWrapper = pMgmt->pWrapper; + pOption->sendReqFp = dndSendReqToDnode; + pOption->sendMnodeReqFp = dndSendReqToMnode; + pOption->sendRspFp = dndSendRsp; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; +} + +static int32_t qmOpenImp(SQnodeMgmt *pMgmt) { + SQnodeOpt option = {0}; + qmInitOption(pMgmt, &option); + + pMgmt->pQnode = qndOpen(&option); + if (pMgmt->pQnode == NULL) { + dError("failed to open qnode since %s", terrstr()); + return -1; + } + + if (qmStartWorker(pMgmt) != 0) { + dError("failed to start qnode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write qnode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void qmCloseImp(SQnodeMgmt *pMgmt) { + if (pMgmt->pQnode != NULL) { + qmStopWorker(pMgmt); + qndClose(pMgmt->pQnode); + pMgmt->pQnode = NULL; + } +} + +int32_t qmDrop(SMgmtWrapper *pWrapper) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("qnode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop qnode since %s", terrstr()); + return -1; + } + + qmCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("qnode-mgmt is dropped"); + return 0; +} + +static void qmClose(SMgmtWrapper *pWrapper) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("qnode-mgmt start to cleanup"); + qmCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("qnode-mgmt is cleaned up"); +} + +int32_t qmOpen(SMgmtWrapper *pWrapper) { + dInfo("qnode-mgmt start to init"); + SQnodeMgmt *pMgmt = calloc(1, sizeof(SQnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = qmOpenImp(pMgmt); + if (code != 0) { + dError("failed to init qnode-mgmt since %s", terrstr()); + qmClose(pWrapper); + } else { + dInfo("qnode-mgmt is initialized"); + } + + return code; +} + +void qmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = qmOpen; + mgmtFp.closeFp = qmClose; + mgmtFp.createMsgFp = qmProcessCreateReq; + mgmtFp.dropMsgFp = qmProcessDropReq; + mgmtFp.requiredFp = qmRequire; + + qmInitMsgHandles(pWrapper); + pWrapper->name = "qnode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/qnode/src/qmMsg.c b/source/dnode/mgmt/qnode/src/qmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..fa7d42b3e34cf737afa32b2c3527398ef2baf048 --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmMsg.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateQnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create qnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return qmOpen(pWrapper); + } +} + +int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropQnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop qnode since %s", terrstr()); + return -1; + } else { + return qmDrop(pWrapper); + } +} + +void qmInitMsgHandles(SMgmtWrapper *pWrapper) {} diff --git a/source/dnode/mgmt/qnode/src/qmWorker.c b/source/dnode/mgmt/qnode/src/qmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..6285a4bb74849491279ff4f623ef84411267e0de --- /dev/null +++ b/source/dnode/mgmt/qnode/src/qmWorker.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "qmInt.h" + +static void qmProcessQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in qnode queue", pMsg); + SRpcMsg *pRsp = NULL; + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = qndProcessMsg(pMgmt->pQnode, pRpc, &pRsp); + + if (pRpc->msgType & 1u) { + if (pRsp != NULL) { + pRsp->ahandle = pRpc->ahandle; + dndSendRsp(pMgmt->pWrapper, pRsp); + free(pRsp); + } else { + if (code != 0) code = terrno; + SRpcMsg rpcRsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; + dndSendRsp(pMgmt->pWrapper, &rpcRsp); + } + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + +int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->queryWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->fetchWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t qmStartWorker(SQnodeMgmt *pMgmt) { + if (dndInitWorker(pMgmt, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, qmProcessQueue) != 0) { + dError("failed to start qnode query worker since %s", terrstr()); + return -1; + } + + if (dndInitWorker(pMgmt, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, qmProcessQueue) != 0) { + dError("failed to start qnode fetch worker since %s", terrstr()); + return -1; + } + + return 0; +} + +void qmStopWorker(SQnodeMgmt *pMgmt) { + dndCleanupWorker(&pMgmt->queryWorker); + dndCleanupWorker(&pMgmt->fetchWorker); +} diff --git a/source/dnode/mgmt/impl/inc/dndSnode.h b/source/dnode/mgmt/snode/inc/sm.h similarity index 68% rename from source/dnode/mgmt/impl/inc/dndSnode.h rename to source/dnode/mgmt/snode/inc/sm.h index b21e9191e8fbedc064f7b9823cec1e68acb70d90..82a52e5d1fdf1577f705a6cebbe86f4f4ecebe27 100644 --- a/source/dnode/mgmt/impl/inc/dndSnode.h +++ b/source/dnode/mgmt/snode/inc/sm.h @@ -16,20 +16,16 @@ #ifndef _TD_DND_SNODE_H_ #define _TD_DND_SNODE_H_ +#include "dnd.h" + #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" - -int32_t dndInitSnode(SDnode *pDnode); -void dndCleanupSnode(SDnode *pDnode); -void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +void smGetMgmtFp(SMgmtWrapper *pWrapper); #ifdef __cplusplus } #endif -#endif /*_TD_DND_SNODE_H_*/ \ No newline at end of file +#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/snode/inc/smInt.h b/source/dnode/mgmt/snode/inc/smInt.h new file mode 100644 index 0000000000000000000000000000000000000000..3def27b83278b8cdb9d174d6403bde28e60be7c2 --- /dev/null +++ b/source/dnode/mgmt/snode/inc/smInt.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_SNODE_INT_H_ +#define _TD_DND_SNODE_INT_H_ + +#include "sm.h" +#include "snode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SSnodeMgmt { + SSnode *pSnode; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + const char *path; + SRWLatch latch; + int8_t uniqueWorkerInUse; + SArray *uniqueWorkers; // SArray + SDnodeWorker sharedWorker; +} SSnodeMgmt; + +// smInt.c +int32_t smOpen(SMgmtWrapper *pWrapper); +int32_t smDrop(SMgmtWrapper *pWrapper); + +// smMsg.c +void smInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); + +// smWorker.c +int32_t smStartWorker(SSnodeMgmt *pMgmt); +void smStopWorker(SSnodeMgmt *pMgmt); +int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_SNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/snode/src/smInt.c b/source/dnode/mgmt/snode/src/smInt.c new file mode 100644 index 0000000000000000000000000000000000000000..0742a4315716695ea320daf448daa88a33417bd5 --- /dev/null +++ b/source/dnode/mgmt/snode/src/smInt.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } + +static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { + SDnode *pDnode = pMgmt->pDnode; + pOption->pWrapper = pMgmt->pWrapper; + pOption->sendReqFp = dndSendReqToDnode; + pOption->sendMnodeReqFp = dndSendReqToMnode; + pOption->sendRspFp = dndSendRsp; + pOption->dnodeId = pDnode->dnodeId; + pOption->clusterId = pDnode->clusterId; +} + +static int32_t smOpenImp(SSnodeMgmt *pMgmt) { + SSnodeOpt option = {0}; + smInitOption(pMgmt, &option); + + pMgmt->pSnode = sndOpen(pMgmt->path, &option); + if (pMgmt->pSnode == NULL) { + dError("failed to open snode since %s", terrstr()); + return -1; + } + + if (smStartWorker(pMgmt) != 0) { + dError("failed to start snode worker since %s", terrstr()); + return -1; + } + + bool deployed = true; + if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) { + dError("failed to write snode file since %s", terrstr()); + return -1; + } + + return 0; +} + +static void smCloseImp(SSnodeMgmt *pMgmt) { + if (pMgmt->pSnode != NULL) { + smStopWorker(pMgmt); + sndClose(pMgmt->pSnode); + pMgmt->pSnode = NULL; + } +} + +int32_t smDrop(SMgmtWrapper *pWrapper) { + SSnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return 0; + + dInfo("snode-mgmt start to drop"); + bool deployed = false; + if (dndWriteFile(pWrapper, deployed) != 0) { + dError("failed to drop snode since %s", terrstr()); + return -1; + } + + smCloseImp(pMgmt); + taosRemoveDir(pMgmt->path); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("snode-mgmt is dropped"); + return 0; +} + +static void smClose(SMgmtWrapper *pWrapper) { + SSnodeMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("snode-mgmt start to cleanup"); + smCloseImp(pMgmt); + pWrapper->pMgmt = NULL; + free(pMgmt); + dInfo("snode-mgmt is cleaned up"); +} + +int32_t smOpen(SMgmtWrapper *pWrapper) { + dInfo("snode-mgmt start to init"); + SSnodeMgmt *pMgmt = calloc(1, sizeof(SSnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + pWrapper->pMgmt = pMgmt; + + int32_t code = smOpenImp(pMgmt); + if (code != 0) { + dError("failed to init snode-mgmt since %s", terrstr()); + smClose(pWrapper); + } else { + dInfo("snode-mgmt is initialized"); + } + + return code; +} + +void smGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = smOpen; + mgmtFp.closeFp = smClose; + mgmtFp.createMsgFp = smProcessCreateReq; + mgmtFp.dropMsgFp = smProcessDropReq; + mgmtFp.requiredFp = smRequire; + + smInitMsgHandles(pWrapper); + pWrapper->name = "snode"; + pWrapper->fp = mgmtFp; +} diff --git a/source/dnode/mgmt/snode/src/smMsg.c b/source/dnode/mgmt/snode/src/smMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..1bff5597bf774504c713c7822e19d3dae9c9b077 --- /dev/null +++ b/source/dnode/mgmt/snode/src/smMsg.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDCreateSnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (createReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to create snode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); + return -1; + } else { + return smOpen(pWrapper); + } +} + +int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnode *pDnode = pWrapper->pDnode; + SRpcMsg *pReq = &pMsg->rpcMsg; + + SDDropSnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (dropReq.dnodeId != pDnode->dnodeId) { + terrno = TSDB_CODE_NODE_INVALID_OPTION; + dError("failed to drop snode since %s", terrstr()); + return -1; + } else { + return smDrop(pWrapper); + } +} + +void smInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by SNODE + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg); +} diff --git a/source/dnode/mgmt/snode/src/smWorker.c b/source/dnode/mgmt/snode/src/smWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..57d0c098497137a2baeb5cf0a87474c8e7ff1edc --- /dev/null +++ b/source/dnode/mgmt/snode/src/smWorker.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "smInt.h" + +static void smProcessUniqueQueue(SSnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) { + for (int32_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + + dTrace("msg:%p, will be processed in snode unique queue", pMsg); + sndProcessUMsg(pMgmt->pSnode, &pMsg->rpcMsg); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } +} + +static void smProcessSharedQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in snode shared queue", pMsg); + sndProcessSMsg(pMgmt->pSnode, &pMsg->rpcMsg); + + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +int32_t smStartWorker(SSnodeMgmt *pMgmt) { + pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *)); + if (pMgmt->uniqueWorkers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { + SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker)); + if (pUniqueWorker == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (dndInitWorker(pMgmt, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, smProcessSharedQueue) != 0) { + dError("failed to start snode unique worker since %s", terrstr()); + return -1; + } + if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + if (dndInitWorker(pMgmt, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM, + SND_SHARED_THREAD_NUM, smProcessSharedQueue)) { + dError("failed to start snode shared worker since %s", terrstr()); + return -1; + } + + return 0; +} + +void smStopWorker(SSnodeMgmt *pMgmt) { + for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { + SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i); + dndCleanupWorker(worker); + } + taosArrayDestroy(pMgmt->uniqueWorkers); + dndCleanupWorker(&pMgmt->sharedWorker); +} + +static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { + SMsgHead *pHead = pMsg->pCont; + pHead->streamTaskId = htonl(pHead->streamTaskId); + return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM; +} + +static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { + SStreamExecMsgHead *pHead = pMsg->pCont; + pHead->workerType = htonl(pHead->workerType); + return pHead->workerType; +} + +int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0); + if (pWorker == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg); + SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index); + if (pWorker == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->sharedWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { + int32_t workerType = smGetSWTypeFromMsg(&pMsg->rpcMsg); + if (workerType == SND_WORKER_TYPE__SHARED) { + return smProcessSharedMsg(pMgmt, pMsg); + } else { + return smProcessUniqueMsg(pMgmt, pMsg); + } +} diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/test/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/CMakeLists.txt rename to source/dnode/mgmt/test/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt b/source/dnode/mgmt/test/bnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/bnode/CMakeLists.txt rename to source/dnode/mgmt/test/bnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp b/source/dnode/mgmt/test/bnode/dbnode.cpp similarity index 88% rename from source/dnode/mgmt/impl/test/bnode/dbnode.cpp rename to source/dnode/mgmt/test/bnode/dbnode.cpp index 75f111587f2c8c8f827f12bff28639cbd3227681..ee81780921ccd09350d6256685a16c302fbfbd05 100644 --- a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp +++ b/source/dnode/mgmt/test/bnode/dbnode.cpp @@ -13,7 +13,10 @@ class DndTestBnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_snode", 9112); } + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_bnode", 9112); + taosMsleep(1100); + } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -36,7 +39,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +65,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -76,11 +79,11 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } -TEST_F(DndTestBnode, 01_Drop_Bnode) { +TEST_F(DndTestBnode, 02_Drop_Bnode) { { SDDropBnodeReq dropReq = {0}; dropReq.dnodeId = 2; @@ -91,7 +94,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -117,7 +120,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -132,7 +135,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { @@ -147,4 +150,4 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/impl/test/mnode/CMakeLists.txt b/source/dnode/mgmt/test/mnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/mnode/CMakeLists.txt rename to source/dnode/mgmt/test/mnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp similarity index 92% rename from source/dnode/mgmt/impl/test/mnode/dmnode.cpp rename to source/dnode/mgmt/test/mnode/dmnode.cpp index 8655bcb774fa1ea0125a3a8cc4755d38b201bb8d..4072eb90a82c3d1239e8d037bc705b95e0464da0 100644 --- a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -40,7 +40,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } { @@ -57,7 +57,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } { @@ -77,7 +77,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -96,7 +96,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -113,7 +113,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -145,7 +145,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -171,7 +171,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { @@ -188,7 +188,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt b/source/dnode/mgmt/test/qnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/qnode/CMakeLists.txt rename to source/dnode/mgmt/test/qnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp similarity index 91% rename from source/dnode/mgmt/impl/test/qnode/dqnode.cpp rename to source/dnode/mgmt/test/qnode/dqnode.cpp index 46c31539a2edd3bde58ba7780cc20132c8a31a0f..343814b15939b60836959674437da47833aa89ab 100644 --- a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -36,7 +36,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +62,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -92,7 +92,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -118,7 +118,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -133,7 +133,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/snode/CMakeLists.txt b/source/dnode/mgmt/test/snode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/snode/CMakeLists.txt rename to source/dnode/mgmt/test/snode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp similarity index 91% rename from source/dnode/mgmt/impl/test/snode/dsnode.cpp rename to source/dnode/mgmt/test/snode/dsnode.cpp index ea98dfdd9558bea777358a40504d451b8bc70872..8dfa437fd8596c2d9d76eef670c7b5cc2fd39411 100644 --- a/source/dnode/mgmt/impl/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -36,7 +36,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -62,7 +62,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); } } @@ -92,7 +92,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_INVALID_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION); } { @@ -118,7 +118,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } test.Restart(); @@ -133,7 +133,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/impl/test/sut/CMakeLists.txt b/source/dnode/mgmt/test/sut/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/sut/CMakeLists.txt rename to source/dnode/mgmt/test/sut/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/sut/inc/client.h b/source/dnode/mgmt/test/sut/inc/client.h similarity index 100% rename from source/dnode/mgmt/impl/test/sut/inc/client.h rename to source/dnode/mgmt/test/sut/inc/client.h diff --git a/source/dnode/mgmt/impl/test/sut/inc/server.h b/source/dnode/mgmt/test/sut/inc/server.h similarity index 86% rename from source/dnode/mgmt/impl/test/sut/inc/server.h rename to source/dnode/mgmt/test/sut/inc/server.h index 99554a7aa78c2e3ffa57ab848a8ccc29c4c7281e..ad2d4a76e91cf9b6d0de787e407fcdd41e4282f7 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/server.h +++ b/source/dnode/mgmt/test/sut/inc/server.h @@ -24,11 +24,11 @@ class TestServer { bool DoStart(); private: - SDnodeObjCfg BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + SDnodeOpt BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); private: SDnode* pDnode; - pthread_t* threadId; + TdThread threadId; char path[PATH_MAX]; char fqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/test/sut/inc/sut.h similarity index 100% rename from source/dnode/mgmt/impl/test/sut/inc/sut.h rename to source/dnode/mgmt/test/sut/inc/sut.h diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/test/sut/src/client.cpp similarity index 100% rename from source/dnode/mgmt/impl/test/sut/src/client.cpp rename to source/dnode/mgmt/test/sut/src/client.cpp diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/test/sut/src/server.cpp similarity index 57% rename from source/dnode/mgmt/impl/test/sut/src/server.cpp rename to source/dnode/mgmt/test/sut/src/server.cpp index 985625b41c28d2518e13a7b01db022b4f276f890..c5379c6d174dce3f4dce5f1cba834e93a5549207 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/test/sut/src/server.cpp @@ -16,36 +16,37 @@ #include "sut.h" void* serverLoop(void* param) { - while (1) { - taosMsleep(100); - pthread_testcancel(); - } + SDnode* pDnode = (SDnode*)param; + dndRun(pDnode); + return NULL; } -SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SDnodeObjCfg cfg = {0}; - cfg.numOfSupportVnodes = 16; - cfg.serverPort = port; - strcpy(cfg.dataDir, path); - snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); - snprintf(cfg.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(cfg.firstEp, TSDB_EP_LEN, "%s", firstEp); - return cfg; +SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SDnodeOpt option = {0}; + option.numOfSupportVnodes = 16; + option.serverPort = port; + strcpy(option.dataDir, path); + snprintf(option.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(option.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); + snprintf(option.firstEp, TSDB_EP_LEN, "%s", firstEp); + return option; } bool TestServer::DoStart() { - SDnodeObjCfg cfg = BuildOption(path, fqdn, port, firstEp); + SDnodeOpt option = BuildOption(path, fqdn, port, firstEp); taosMkDir(path); - pDnode = dndCreate(&cfg); - if (pDnode != NULL) { + pDnode = dndCreate(&option); + if (pDnode == NULL) { return false; } - threadId = taosCreateThread(serverLoop, NULL); - if (threadId != NULL) { - return false; - } + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&threadId, &thAttr, serverLoop, pDnode); + taosThreadAttrDestroy(&thAttr); + taosMsleep(2100); return true; } @@ -67,10 +68,8 @@ bool TestServer::Start(const char* path, const char* fqdn, uint16_t port, const } void TestServer::Stop() { - if (threadId != NULL) { - taosDestoryThread(threadId); - threadId = NULL; - } + dndHandleEvent(pDnode, DND_EVENT_STOP); + taosThreadJoin(threadId, NULL); if (pDnode != NULL) { dndClose(pDnode); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp similarity index 98% rename from source/dnode/mgmt/impl/test/sut/src/sut.cpp rename to source/dnode/mgmt/test/sut/src/sut.cpp index 2e38ea75137f4120b4611db8c6f133164e05cee2..14197153b49bb2552d29409801d35ce40d8090c6 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -21,8 +21,8 @@ void Testbase::InitLog(const char* path) { mDebugFlag = 143; cDebugFlag = 0; jniDebugFlag = 0; - tmrDebugFlag = 143; - uDebugFlag = 143; + tmrDebugFlag = 135; + uDebugFlag = 135; rpcDebugFlag = 143; qDebugFlag = 0; wDebugFlag = 0; @@ -49,7 +49,6 @@ void Testbase::Init(const char* path, int16_t port) { InitLog("/tmp/td"); server.Start(path, fqdn, port, firstEp); client.Init("root", "taosdata", fqdn, port); - taosMsleep(1100); tFreeSTableMetaRsp(&metaRsp); showId = 0; diff --git a/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt b/source/dnode/mgmt/test/vnode/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/vnode/CMakeLists.txt rename to source/dnode/mgmt/test/vnode/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp similarity index 100% rename from source/dnode/mgmt/impl/test/vnode/vnode.cpp rename to source/dnode/mgmt/test/vnode/vnode.cpp diff --git a/source/dnode/mgmt/impl/inc/dndQnode.h b/source/dnode/mgmt/vnode/inc/vm.h similarity index 53% rename from source/dnode/mgmt/impl/inc/dndQnode.h rename to source/dnode/mgmt/vnode/inc/vm.h index 2a25dca1c6dc324f3d98d425e5962df0a410aa13..60d9cfc3a175344014870a103078555c22cc569c 100644 --- a/source/dnode/mgmt/impl/inc/dndQnode.h +++ b/source/dnode/mgmt/vnode/inc/vm.h @@ -13,24 +13,34 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_QNODE_H_ -#define _TD_DND_QNODE_H_ +#ifndef _TD_DND_VNODES_H_ +#define _TD_DND_VNODES_H_ + +#include "dnd.h" #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitQnode(SDnode *pDnode); -void dndCleanupQnode(SDnode *pDnode); +typedef struct { + int32_t openVnodes; + int32_t totalVnodes; + int32_t masterNum; + int64_t numOfSelectReqs; + int64_t numOfInsertReqs; + int64_t numOfInsertSuccessReqs; + int64_t numOfBatchInsertReqs; + int64_t numOfBatchInsertSuccessReqs; +} SVnodesStat; + +void vmGetMgmtFp(SMgmtWrapper *pWrapper); -void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); #ifdef __cplusplus } #endif -#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file +#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/inc/vmInt.h b/source/dnode/mgmt/vnode/inc/vmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..c0e7e212cc98853bfdaa58be474ed0e6798549a1 --- /dev/null +++ b/source/dnode/mgmt/vnode/inc/vmInt.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_VNODES_INT_H_ +#define _TD_DND_VNODES_INT_H_ + +#include "sync.h" +#include "vm.h" +#include "vnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SVnodesMgmt { + SHashObj *hash; + SRWLatch latch; + SVnodesStat state; + STfs *pTfs; + SQWorkerPool queryPool; + SFWorkerPool fetchPool; + SWWorkerPool syncPool; + SWWorkerPool writePool; + const char *path; + SDnode *pDnode; + SMgmtWrapper *pWrapper; + SDnodeWorker mgmtWorker; +} SVnodesMgmt; + +typedef struct { + int32_t vgId; + int32_t vgVersion; + int8_t dropped; + uint64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; + char path[PATH_MAX + 20]; +} SWrapperCfg; + +typedef struct { + int32_t vgId; + int32_t refCount; + int32_t vgVersion; + int8_t dropped; + int8_t accessState; + uint64_t dbUid; + char *db; + char *path; + SVnode *pImpl; + STaosQueue *pWriteQ; + STaosQueue *pSyncQ; + STaosQueue *pApplyQ; + STaosQueue *pQueryQ; + STaosQueue *pFetchQ; + SMgmtWrapper *pWrapper; +} SVnodeObj; + +typedef struct { + int32_t vnodeNum; + int32_t opened; + int32_t failed; + int32_t threadIndex; + TdThread thread; + SVnodesMgmt *pMgmt; + SWrapperCfg *pCfgs; +} SVnodeThread; + +// vmInt.c +SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId); +void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); +int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); +void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); + +// vmMsg.c +void vmInitMsgHandles(SMgmtWrapper *pWrapper); +int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); + +// vmFile.c +int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); +int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt); +SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes); + +// vmWorker.c +int32_t vmStartWorker(SVnodesMgmt *pMgmt); +void vmStopWorker(SVnodesMgmt *pMgmt); +int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); +void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); + +int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, int32_t vgId, SRpcMsg *pMsg); + +int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_VNODES_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/src/vmFile.c b/source/dnode/mgmt/vnode/src/vmFile.c new file mode 100644 index 0000000000000000000000000000000000000000..0fe868dfa41247c0a34ff293d81f11a93c182a01 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmFile.c @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes) { + taosRLockLatch(&pMgmt->latch); + + int32_t num = 0; + int32_t size = taosHashGetSize(pMgmt->hash); + SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + SVnodeObj *pVnode = *ppVnode; + if (pVnode && num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + pVnodes[num] = (*ppVnode); + num++; + pIter = taosHashIterate(pMgmt->hash, pIter); + } else { + taosHashCancelIterate(pMgmt->hash, pIter); + } + } + + taosRUnLockLatch(&pMgmt->latch); + *numOfVnodes = num; + + return pVnodes; +} + +int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 30000; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; + char file[PATH_MAX]; + SWrapperCfg *pCfgs = NULL; + TdFilePtr pFile = NULL; + + snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_VNODE_OVER; + } + + len = (int32_t)taosReadFile(pFile, content, maxLen); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_VNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_VNODE_OVER; + } + + cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); + if (!vnodes || vnodes->type != cJSON_Array) { + dError("failed to read %s since vnodes not found", file); + goto PRASE_VNODE_OVER; + } + + int32_t vnodesNum = cJSON_GetArraySize(vnodes); + if (vnodesNum > 0) { + pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); + if (pCfgs == NULL) { + dError("failed to read %s since out of memory", file); + goto PRASE_VNODE_OVER; + } + + for (int32_t i = 0; i < vnodesNum; ++i) { + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + SWrapperCfg *pCfg = &pCfgs[i]; + + cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); + if (!vgId || vgId->type != cJSON_Number) { + dError("failed to read %s since vgId not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgId = vgId->valueint; + snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); + + cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dropped = dropped->valueint; + + cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); + if (!vgVersion || vgVersion->type != cJSON_Number) { + dError("failed to read %s since vgVersion not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgVersion = vgVersion->valueint; + + cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); + if (!dbUid || dbUid->type != cJSON_String) { + dError("failed to read %s since dbUid not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dbUid = atoll(dbUid->valuestring); + + cJSON *db = cJSON_GetObjectItem(vnode, "db"); + if (!db || db->type != cJSON_String) { + dError("failed to read %s since db not found", file); + goto PRASE_VNODE_OVER; + } + tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN); + } + + *ppCfgs = pCfgs; + } + + *numOfVnodes = vnodesNum; + code = 0; + dInfo("succcessed to read file %s", file); + +PRASE_VNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (pFile != NULL) taosCloseFile(&pFile); + + terrno = code; + return code; +} + +int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { + char file[PATH_MAX]; + char realfile[PATH_MAX]; + snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); + snprintf(realfile, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = vmGetVnodesFromHash(pMgmt, &numOfVnodes); + + int32_t len = 0; + int32_t maxLen = 65536; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + len += snprintf(content + len, maxLen - len, " {\n"); + len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); + len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); + if (i < numOfVnodes - 1) { + len += snprintf(content + len, maxLen - len, " },\n"); + } else { + len += snprintf(content + len, maxLen - len, " }\n"); + } + } + len += snprintf(content + len, maxLen - len, " ]\n"); + len += snprintf(content + len, maxLen - len, "}\n"); + + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + free(content); + terrno = 0; + + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + vmReleaseVnode(pMgmt, pVnode); + } + + if (pVnodes != NULL) { + free(pVnodes); + } + + dDebug("successed to write %s", realfile); + return taosRenameFile(file, realfile); +} \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/src/vmInt.c b/source/dnode/mgmt/vnode/src/vmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..8b2c2d7dd44e015f4ce5884d4b17fc8a6bf81d21 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmInt.c @@ -0,0 +1,410 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { + SVnodeObj *pVnode = NULL; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + if (pVnode == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + } else { + refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + if (pVnode != NULL) { + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + } + + return pVnode; +} + +void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + if (pVnode == NULL) return; + + taosRLockLatch(&pMgmt->latch); + int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + taosRUnLockLatch(&pMgmt->latch); + dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); +} + +int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { + SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); + if (pVnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pVnode->vgId = pCfg->vgId; + pVnode->refCount = 0; + pVnode->dropped = 0; + pVnode->accessState = TSDB_VN_ALL_ACCCESS; + pVnode->pWrapper = pMgmt->pWrapper; + pVnode->pImpl = pImpl; + pVnode->vgVersion = pCfg->vgVersion; + pVnode->dbUid = pCfg->dbUid; + pVnode->db = tstrdup(pCfg->db); + pVnode->path = tstrdup(pCfg->path); + + if (pVnode->path == NULL || pVnode->db == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (vmAllocQueue(pMgmt, pVnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + taosWLockLatch(&pMgmt->latch); + int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); + taosWUnLockLatch(&pMgmt->latch); + + if (code != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return code; +} + +void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + taosWLockLatch(&pMgmt->latch); + taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); + taosWUnLockLatch(&pMgmt->latch); + + vmReleaseVnode(pMgmt, pVnode); + while (pVnode->refCount > 0) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + + vmFreeQueue(pMgmt, pVnode); + vnodeClose(pVnode->pImpl); + pVnode->pImpl = NULL; + + dDebug("vgId:%d, vnode is closed", pVnode->vgId); + + if (pVnode->dropped) { + dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped); + vnodeDestroy(pVnode->path); + } + + free(pVnode->path); + free(pVnode->db); + free(pVnode); +} + +static void *vmOpenVnodeFunc(void *param) { + SVnodeThread *pThread = param; + SVnodesMgmt *pMgmt = pThread->pMgmt; + SDnode *pDnode = pMgmt->pDnode; + + dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + setThreadName("open-vnodes"); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + SWrapperCfg *pCfg = &pThread->pCfgs[v]; + + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, + pMgmt->state.openVnodes, pMgmt->state.totalVnodes); + dndReportStartup(pDnode, "open-vnodes", stepDesc); + + SVnodeCfg cfg = {.pWrapper = pMgmt->pWrapper, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; + SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); + if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); + pThread->failed++; + } else { + vmOpenVnode(pMgmt, pCfg, pImpl); + dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); + pThread->opened++; + } + + atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); + } + + dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { + SDnode *pDnode = pMgmt->pDnode; + taosInitRWLatch(&pMgmt->latch); + + pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMgmt->hash == NULL) { + dError("failed to init vnode hash"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SWrapperCfg *pCfgs = NULL; + int32_t numOfVnodes = 0; + if (vmGetVnodesFromFile(pMgmt, &pCfgs, &numOfVnodes) != 0) { + dInfo("failed to get vnode list from disk since %s", terrstr()); + return -1; + } + + pMgmt->state.totalVnodes = numOfVnodes; + +#if 0 + int32_t threadNum = tsNumOfCores; +#else + int32_t threadNum = 1; +#endif + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + + SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].pMgmt = pMgmt; + threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SVnodeThread *pThread = &threads[t]; + pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; + } + + dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pThread->thread, &thAttr, vmOpenVnodeFunc, pThread) != 0) { + dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + } + + taosThreadAttrDestroy(&thAttr); + } + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + taosThreadJoin(pThread->thread, NULL); + } + free(pThread->pCfgs); + } + free(threads); + free(pCfgs); + + if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { + dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); + return -1; + } else { + dInfo("total vnodes:%d open successfully", pMgmt->state.totalVnodes); + return 0; + } +} + +static void vmCloseVnodes(SVnodesMgmt *pMgmt) { + dInfo("start to close all vnodes"); + + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = vmGetVnodesFromHash(pMgmt, &numOfVnodes); + + for (int32_t i = 0; i < numOfVnodes; ++i) { + vmCloseVnode(pMgmt, pVnodes[i]); + } + + if (pVnodes != NULL) { + free(pVnodes); + } + + if (pMgmt->hash != NULL) { + taosHashCleanup(pMgmt->hash); + pMgmt->hash = NULL; + } + + dInfo("total vnodes:%d are all closed", numOfVnodes); +} + +static void vmCleanup(SMgmtWrapper *pWrapper) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + dInfo("vnodes-mgmt start to cleanup"); + vmCloseVnodes(pMgmt); + vmStopWorker(pMgmt); + vnodeCleanup(); + // walCleanUp(); + free(pMgmt); + pWrapper->pMgmt = NULL; + dInfo("vnodes-mgmt is cleaned up"); +} + +static int32_t vmInit(SMgmtWrapper *pWrapper) { + SDnode *pDnode = pWrapper->pDnode; + SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt)); + int32_t code = -1; + SVnodeOpt vnodeOpt = {0}; + + dInfo("vnodes-mgmt start to init"); + if (pMgmt == NULL) goto _OVER; + + pMgmt->path = pWrapper->path; + pMgmt->pDnode = pWrapper->pDnode; + pMgmt->pWrapper = pWrapper; + taosInitRWLatch(&pMgmt->latch); + + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, pDnode->dataDir, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + SDiskCfg *pDisks = pDnode->pDisks; + int32_t numOfDisks = pDnode->numOfDisks; + if (numOfDisks <= 0 || pDisks == NULL) { + pDisks = &dCfg; + numOfDisks = 1; + } + + pMgmt->pTfs = tfsOpen(pDisks, numOfDisks); + if (pMgmt->pTfs == NULL) { + dError("failed to init tfs since %s", terrstr()); + goto _OVER; + } + + if (walInit() != 0) { + dError("failed to init wal since %s", terrstr()); + goto _OVER; + } + + vnodeOpt.nthreads = tsNumOfCommitThreads; + vnodeOpt.putToQueryQFp = vmPutMsgToQueryQueue; + vnodeOpt.sendReqFp = dndSendReqToDnode; + if (vnodeInit(&vnodeOpt) != 0) { + dError("failed to init vnode since %s", terrstr()); + goto _OVER; + } + + if (vmStartWorker(pMgmt) != 0) { + dError("failed to init workers since %s", terrstr()) goto _OVER; + } + + if (vmOpenVnodes(pMgmt) != 0) { + dError("failed to open vnodes since %s", terrstr()); + return -1; + } + + code = 0; + +_OVER: + if (code == 0) { + pWrapper->pMgmt = pMgmt; + dInfo("vnodes-mgmt is initialized"); + } else { + dError("failed to init vnodes-mgmt since %s", terrstr()); + vmCleanup(pWrapper); + } + + return 0; +} + +static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { + SDnode *pDnode = pWrapper->pDnode; + *required = pDnode->numOfSupportVnodes > 0; + return 0; +} + +void vmGetMgmtFp(SMgmtWrapper *pWrapper) { + SMgmtFp mgmtFp = {0}; + mgmtFp.openFp = vmInit; + mgmtFp.closeFp = vmCleanup; + mgmtFp.requiredFp = vmRequire; + + vmInitMsgHandles(pWrapper); + pWrapper->name = "vnodes"; + pWrapper->fp = mgmtFp; +} + +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return -1; + + return tfsGetMonitorInfo(pMgmt->pTfs, pInfo); +} + +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + if (pMgmt == NULL) return; + + SVnodesStat *pStat = &pMgmt->state; + pInfo->req_select = pStat->numOfSelectReqs; + pInfo->req_insert = pStat->numOfInsertReqs; + pInfo->req_insert_success = pStat->numOfInsertSuccessReqs; + pInfo->req_insert_batch = pStat->numOfBatchInsertReqs; + pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs; + pInfo->errors = tsNumOfErrorLogs; + pInfo->vnodes_num = pStat->totalVnodes; + pInfo->masters = pStat->masterNum; +} + +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + SVnodesStat *pStat = &pMgmt->state; + int32_t totalVnodes = 0; + int32_t masterNum = 0; + int64_t numOfSelectReqs = 0; + int64_t numOfInsertReqs = 0; + int64_t numOfInsertSuccessReqs = 0; + int64_t numOfBatchInsertReqs = 0; + int64_t numOfBatchInsertSuccessReqs = 0; + + taosRLockLatch(&pMgmt->latch); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + if (ppVnode == NULL || *ppVnode == NULL) continue; + + SVnodeObj *pVnode = *ppVnode; + SVnodeLoad vload = {0}; + vnodeGetLoad(pVnode->pImpl, &vload); + taosArrayPush(pLoads, &vload); + + numOfSelectReqs += vload.numOfSelectReqs; + numOfInsertReqs += vload.numOfInsertReqs; + numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs; + numOfBatchInsertReqs += vload.numOfBatchInsertReqs; + numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs; + totalVnodes++; + if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++; + + pIter = taosHashIterate(pMgmt->hash, pIter); + } + + taosRUnLockLatch(&pMgmt->latch); + + pStat->totalVnodes = totalVnodes; + pStat->masterNum = masterNum; + pStat->numOfSelectReqs = numOfSelectReqs; + pStat->numOfInsertReqs = numOfInsertReqs; + pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs; + pStat->numOfBatchInsertReqs = numOfBatchInsertReqs; + pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; +} \ No newline at end of file diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..b3f752931177c380bd216fb501b319cb35e7dede --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { + pCfg->vgId = pCreate->vgId; + pCfg->wsize = pCreate->cacheBlockSize; + pCfg->ssize = pCreate->cacheBlockSize; + pCfg->lsize = pCreate->cacheBlockSize; + pCfg->isHeapAllocator = true; + pCfg->ttl = 4; + pCfg->keep = pCreate->daysToKeep0; + pCfg->streamMode = pCreate->streamMode; + pCfg->isWeak = true; + pCfg->tsdbCfg.keep = pCreate->daysToKeep0; + pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; + pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; + pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; + pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; + pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; + pCfg->walCfg.level = pCreate->walLevel; + pCfg->walCfg.retentionPeriod = 10; + pCfg->walCfg.retentionSize = 128; + pCfg->walCfg.rollPeriod = 128; + pCfg->walCfg.segSize = 128; + pCfg->walCfg.vgId = pCreate->vgId; + pCfg->hashBegin = pCreate->hashBegin; + pCfg->hashEnd = pCreate->hashEnd; + pCfg->hashMethod = pCreate->hashMethod; +} + +static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { + memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN); + pCfg->dbUid = pCreate->dbUid; + pCfg->dropped = 0; + snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCreate->vgId); + pCfg->vgId = pCreate->vgId; + pCfg->vgVersion = pCreate->vgVersion; +} + +int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SCreateVnodeReq createReq = {0}; + if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dDebug("vgId:%d, create vnode req is received", createReq.vgId); + + SVnodeCfg vnodeCfg = {0}; + vmGenerateVnodeCfg(&createReq, &vnodeCfg); + + SWrapperCfg wrapperCfg = {0}; + vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); + + if (createReq.dnodeId != pMgmt->pDnode->dnodeId) { + terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; + dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + return -1; + } + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); + if (pVnode != NULL) { + dDebug("vgId:%d, already exist", createReq.vgId); + vmReleaseVnode(pMgmt, pVnode); + terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; + return -1; + } + + vnodeCfg.pWrapper = pMgmt->pWrapper; + vnodeCfg.pTfs = pMgmt->pTfs; + vnodeCfg.dbId = wrapperCfg.dbUid; + SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); + if (pImpl == NULL) { + dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + return -1; + } + + int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl); + if (code != 0) { + dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + code = vmWriteVnodesToFile(pMgmt); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + return 0; +} + +int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SAlterVnodeReq alterReq = {0}; + if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dDebug("vgId:%d, alter vnode req is received", alterReq.vgId); + + SVnodeCfg vnodeCfg = {0}; + vmGenerateVnodeCfg(&alterReq, &vnodeCfg); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, alterReq.vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); + return -1; + } + + if (alterReq.vgVersion == pVnode->vgVersion) { + vmReleaseVnode(pMgmt, pVnode); + dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId); + return 0; + } + + if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { + dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + int32_t oldVersion = pVnode->vgVersion; + pVnode->vgVersion = alterReq.vgVersion; + int32_t code = vmWriteVnodesToFile(pMgmt); + if (code != 0) { + pVnode->vgVersion = oldVersion; + } + + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SDropVnodeReq dropReq = {0}; + if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t vgId = dropReq.vgId; + dDebug("vgId:%d, drop vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); + terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; + return -1; + } + + pVnode->dropped = 1; + if (vmWriteVnodesToFile(pMgmt) != 0) { + pVnode->dropped = 0; + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmCloseVnode(pMgmt, pVnode); + vmWriteVnodesToFile(pMgmt); + + return 0; +} + +int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SSyncVnodeReq syncReq = {0}; + tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq); + + int32_t vgId = syncReq.vgId; + dDebug("vgId:%d, sync vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); + return -1; + } + + if (vnodeSync(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmReleaseVnode(pMgmt, pVnode); + return 0; +} + +int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SCompactVnodeReq compatcReq = {0}; + tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq); + + int32_t vgId = compatcReq.vgId; + dDebug("vgId:%d, compact vnode req is received", vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); + return -1; + } + + if (vnodeCompact(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + vmReleaseVnode(pMgmt, pVnode); + return 0; +} + +void vmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by VNODE + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg); + + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg); +} diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..e5d5e38328528b3d4bd5bcd34ba24b068debe0e9 --- /dev/null +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vmInt.h" + +static void vmProcessQueryQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in vnode query queue", pMsg); + vnodeProcessQueryMsg(pVnode->pImpl, &pMsg->rpcMsg); +} + +static void vmProcessFetchQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in vnode fetch queue", pMsg); + vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg); +} + +static void vmProcessWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); + + for (int32_t i = 0; i < numOfMsgs; ++i) { + SNodeMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + dTrace("msg:%p, will be processed in vnode write queue", pMsg); + void *ptr = taosArrayPush(pArray, &pMsg); + assert(ptr != NULL); + } + + vnodeProcessWMsgs(pVnode->pImpl, pArray); + + for (size_t i = 0; i < numOfMsgs; i++) { + SRpcMsg *pRsp = NULL; + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp); + if (pRsp != NULL) { + pRsp->ahandle = pRpc->ahandle; + dndSendRsp(pVnode->pWrapper, pRsp); + free(pRsp); + } else { + if (code != 0) code = terrno; + SRpcMsg rpcRsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; + dndSendRsp(pVnode->pWrapper, &rpcRsp); + } + } + + for (size_t i = 0; i < numOfMsgs; i++) { + SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); + dTrace("msg:%p, is freed", pMsg); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } + + taosArrayDestroy(pArray); +} + +static void vmProcessApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { + SNodeMsg *pMsg = NULL; + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + + // todo + SRpcMsg *pRsp = NULL; + (void)vnodeApplyWMsg(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + } +} + +static void vmProcessSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { + SNodeMsg *pMsg = NULL; + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + + // todo + SRpcMsg *pRsp = NULL; + (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + } +} + +static SVnodeObj *vmAcquireFromMsg(SVnodesMgmt *pMgmt, SNodeMsg *pNodeMsg) { + SRpcMsg *pMsg = &pNodeMsg->rpcMsg; + + SMsgHead *pHead = pMsg->pCont; + pHead->contLen = htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); + if (pVnode == NULL) { + dError("vgId:%d, failed to acquire vnode while process req", pHead->vgId); + } + + return pVnode; +} + +int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SVnodeObj *pVnode = vmAcquireFromMsg(pMgmt, pMsg); + if (pVnode == NULL) return -1; + + int32_t code = taosWriteQitem(pVnode->pWriteQ, pMsg); + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SVnodeObj *pVnode = vmAcquireFromMsg(pMgmt, pMsg); + if (pVnode == NULL) return -1; + + int32_t code = taosWriteQitem(pVnode->pSyncQ, pMsg); + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SVnodeObj *pVnode = vmAcquireFromMsg(pMgmt, pMsg); + if (pVnode == NULL) return -1; + + int32_t code = taosWriteQitem(pVnode->pQueryQ, pMsg); + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SVnodeObj *pVnode = vmAcquireFromMsg(pMgmt, pMsg); + if (pVnode == NULL) return -1; + + int32_t code = taosWriteQitem(pVnode->pFetchQ, pMsg); + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + + int32_t code = -1; + SMsgHead *pHead = pRpc->pCont; + // pHead->vgId = htonl(pHead->vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); + if (pVnode == NULL) return -1; + + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg != NULL) { + pMsg->rpcMsg = *pRpc; + code = taosWriteQitem(pVnode->pQueryQ, pMsg); + } + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, int32_t vgId, SRpcMsg *pRpc) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + + int32_t code = -1; + SMsgHead *pHead = pRpc->pCont; + // pHead->vgId = htonl(pHead->vgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); + if (pVnode == NULL) return -1; + + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg != NULL) { + pMsg->rpcMsg = *pRpc; + code = taosWriteQitem(pVnode->pApplyQ, pMsg); + } + vmReleaseVnode(pMgmt, pVnode); + return code; +} + +int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); + pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); + pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); + pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); + pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); + + if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || + pVnode->pQueryQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { + tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); + tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); + pVnode->pWriteQ = NULL; + pVnode->pApplyQ = NULL; + pVnode->pSyncQ = NULL; + pVnode->pFetchQ = NULL; + pVnode->pQueryQ = NULL; +} + +static void vmProcessMgmtQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + int32_t code = -1; + tmsg_t msgType = pMsg->rpcMsg.msgType; + dTrace("msg:%p, will be processed in vnode mgmt queue", pMsg); + + switch (msgType) { + case TDMT_DND_CREATE_VNODE: + code = vmProcessCreateVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_ALTER_VNODE: + code = vmProcessAlterVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_DROP_VNODE: + code = vmProcessDropVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_SYNC_VNODE: + code = vmProcessSyncVnodeReq(pMgmt, pMsg); + break; + case TDMT_DND_COMPACT_VNODE: + code = vmProcessCompactVnodeReq(pMgmt, pMsg); + break; + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + dError("msg:%p, not processed in mgmt queue", pMsg); + } + + if (msgType & 1u) { + if (code != 0) code = terrno; + SRpcMsg rsp = {.code = code, .handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle}; + dndSendRsp(pMgmt->pWrapper, &rsp); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +int32_t vmStartWorker(SVnodesMgmt *pMgmt) { + int32_t maxFetchThreads = 4; + int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); + int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); + int32_t maxQueryThreads = minQueryThreads; + int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); + int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); + + SQWorkerPool *pQPool = &pMgmt->queryPool; + pQPool->name = "vnode-query"; + pQPool->min = minQueryThreads; + pQPool->max = maxQueryThreads; + if (tQWorkerInit(pQPool) != 0) return -1; + + SFWorkerPool *pFPool = &pMgmt->fetchPool; + pFPool->name = "vnode-fetch"; + pFPool->min = minFetchThreads; + pFPool->max = maxFetchThreads; + if (tFWorkerInit(pFPool) != 0) return -1; + + SWWorkerPool *pWPool = &pMgmt->writePool; + pWPool->name = "vnode-write"; + pWPool->max = maxWriteThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + + pWPool = &pMgmt->syncPool; + pWPool->name = "vnode-sync"; + pWPool->max = maxSyncThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + + if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "vnode-mgmt", 1, 1, vmProcessMgmtQueue) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); + return -1; + } + + dDebug("vnode workers is initialized"); + return 0; +} + +void vmStopWorker(SVnodesMgmt *pMgmt) { + dndCleanupWorker(&pMgmt->mgmtWorker); + tFWorkerCleanup(&pMgmt->fetchPool); + tQWorkerCleanup(&pMgmt->queryPool); + tWWorkerCleanup(&pMgmt->writePool); + tWWorkerCleanup(&pMgmt->syncPool); + dDebug("vnode workers is closed"); +} + +int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { + SDnodeWorker *pWorker = &pMgmt->mgmtWorker; + dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9ed6104140ab0b475e27daf60311a30a383b4e88..909486aaac211b405a3fc468fc151f9b46573648 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -85,6 +85,8 @@ typedef enum { TRN_TYPE_REBALANCE = 1017, TRN_TYPE_COMMIT_OFFSET = 1018, TRN_TYPE_CREATE_STREAM = 1019, + TRN_TYPE_DROP_STREAM = 1020, + TRN_TYPE_ALTER_STREAM = 1021, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -687,27 +689,19 @@ typedef struct { int64_t uid; int64_t dbUid; int32_t version; + int32_t vgNum; SRWLatch lock; int8_t status; // int32_t sqlLen; - char* sql; - char* logicalPlan; - char* physicalPlan; + char* sql; + char* logicalPlan; + char* physicalPlan; + SArray* tasks; // SArray> } SStreamObj; int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj); -typedef struct SMnodeMsg { - char user[TSDB_USER_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int32_t acctId; - SMnode* pMnode; - int64_t createdTime; - SRpcMsg rpcMsg; - int32_t contLen; - void* pCont; -} SMnodeMsg; #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 0fe24cd7575162a2ecb1bfe41e77ed6df49e2060..88dd3a5c22c534d186264c555a7097dc8aaa8664 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -38,11 +38,11 @@ extern "C" { #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} -typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); +typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); -typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +typedef int32_t (*ShowMetaFp)(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct SMnodeLoad { @@ -110,7 +110,7 @@ typedef struct SMnode { char *path; int64_t checkTime; SSdb *pSdb; - SDnode *pDnode; + SMgmtWrapper *pWrapper; SArray *pSteps; SShowMgmt showMgmt; SProfileMgmt profileMgmt; @@ -119,16 +119,14 @@ typedef struct SMnode { SHashObj *infosMeta; SGrantInfo grant; MndMsgFp msgFp[TDMT_MAX]; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; - PutReqToMWriteQFp putReqToMWriteQFp; - PutReqToMReadQFp putReqToMReadQFp; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + PutToQueueFp putToWriteQFp; + PutToQueueFp putToReadQFp; } SMnode; int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg); int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg); -void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg); void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); uint64_t mndGenerateUid(char *name, int32_t len); diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 3bf6e0c33a6a07c633c38323c133099fed62e17b..42951beca2e414611543d92e08b86d19f6247636 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -27,6 +27,8 @@ void mndCleanupScheduler(SMnode* pMnode); int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub); +int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index a0218eaf65807a571de7271b55a4ca4ccacf891a..5c1b0991be803e765637c8f36b3b84cdb1fc7056 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -47,7 +47,7 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); -void mndTransProcessRsp(SMnodeMsg *pRsp); +void mndTransProcessRsp(SNodeMsg *pRsp); void mndTransPullup(SMnode *pMnode); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index aa87eb43a18a84e04bef9056d3fde8991daa6d2a..a6e6d57345ceca265575bb075b58825489eb4d14 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -26,9 +26,9 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw); static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew); -static int32_t mndProcessCreateAcctReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterAcctReq(SMnodeMsg *pReq); -static int32_t mndProcessDropAcctReq(SMnodeMsg *pReq); +static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq); +static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq); +static int32_t mndProcessDropAcctReq(SNodeMsg *pReq); int32_t mndInitAcct(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_ACCT, @@ -185,19 +185,19 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { return 0; } -static int32_t mndProcessCreateAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } -static int32_t mndProcessAlterAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } -static int32_t mndProcessDropAcctReq(SMnodeMsg *pReq) { +static int32_t mndProcessDropAcctReq(SNodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index ab8b9350b099e01d646b7fb9a43976f10fe93ad3..20dd7f1378ff857dd21c63cb6472668ce4ff2556 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -17,7 +17,7 @@ #include "mndAuth.h" #include "mndUser.h" -static int32_t mndProcessAuthReq(SMnodeMsg *pReq); +static int32_t mndProcessAuthReq(SNodeMsg *pReq); int32_t mndInitAuth(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq); @@ -45,7 +45,7 @@ int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, cha return 0; } -static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { +static int32_t mndProcessAuthReq(SNodeMsg *pReq) { SAuthReq authReq = {0}; if (tDeserializeSAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -56,15 +56,15 @@ static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { memcpy(authRsp.user, authReq.user, TSDB_USER_LEN); int32_t code = - mndRetriveAuth(pReq->pMnode, authRsp.user, &authRsp.spi, &authRsp.encrypt, authRsp.secret, authRsp.ckey); + mndRetriveAuth(pReq->pNode, authRsp.user, &authRsp.spi, &authRsp.encrypt, authRsp.secret, authRsp.ckey); mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->user, authRsp.spi, authRsp.encrypt, authRsp.user); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp); void *pRsp = rpcMallocCont(contLen); tSerializeSAuthReq(pRsp, contLen, &authRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; return code; } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index a9921354644c4837305767d0a2f1e2ef568b2654..d9a0df555c33287eeae890c538d17cc6da2ff8ed 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -29,12 +29,12 @@ static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); -static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateBnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropBnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateBnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetBnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); int32_t mndInitBnode(SMnode *pMnode) { @@ -203,7 +203,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -230,7 +230,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -240,7 +240,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } -static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { +static int32_t mndCreateBnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { int32_t code = -1; SBnodeObj bnodeObj = {0}; @@ -266,8 +266,8 @@ CREATE_BNODE_OVER: return code; } -static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateBnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SBnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -353,7 +353,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -363,7 +363,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn return 0; } -static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pReq, SBnodeObj *pObj) { +static int32_t mndDropBnode(SMnode *pMnode, SNodeMsg *pReq, SBnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_BNODE, &pReq->rpcMsg); @@ -382,8 +382,8 @@ DROP_BNODE_OVER: return code; } -static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropBnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SBnodeObj *pObj = NULL; @@ -430,18 +430,18 @@ DROP_BNODE_OVER: return code; } -static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateBnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetBnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -480,8 +480,8 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 3410a386da2d80bafcb35db5f2d97e987f753eb3..ce470806452b4ac50c7e6f8e9e1d78755619c466 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -26,8 +26,8 @@ static int32_t mndClusterActionInsert(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOldCluster, SClusterObj *pNewCluster); static int32_t mndCreateDefaultCluster(SMnode *pMnode); -static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetClusterMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter); int32_t mndInitCluster(SMnode *pMnode) { @@ -163,7 +163,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } -static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { +static int32_t mndGetClusterMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchemas; @@ -201,8 +201,8 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 0f4b538cde51b85a65089a42072a799b6c8dccc0..d86665b832e63ed59943b3500c723f036bb55d60 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -34,9 +34,9 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); -static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessConsumerMetaMsg(SNodeMsg *pMsg); +static int32_t mndGetConsumerMeta(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConsumer(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); int32_t mndInitConsumer(SMnode *pMnode) { @@ -96,12 +96,12 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { CM_ENCODE_OVER: tfree(buf); if (terrno != 0) { - mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); + mError("consumer:%" PRId64 ", failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); sdbFreeRaw(pRaw); return NULL; } - mTrace("consumer:%ld, encode to raw:%p, row:%p", pConsumer->consumerId, pRaw, pConsumer); + mTrace("consumer:%" PRId64 ", encode to raw:%p, row:%p", pConsumer->consumerId, pRaw, pConsumer); return pRaw; } @@ -140,7 +140,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { CM_DECODE_OVER: tfree(buf); if (terrno != TSDB_CODE_SUCCESS) { - mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); + mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); tfree(pRow); return NULL; } @@ -149,17 +149,17 @@ CM_DECODE_OVER: } static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mTrace("consumer:%ld, perform insert action", pConsumer->consumerId); + mTrace("consumer:%" PRId64 ", perform insert action", pConsumer->consumerId); return 0; } static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mTrace("consumer:%ld, perform delete action", pConsumer->consumerId); + mTrace("consumer:%" PRId64 ", perform delete action", pConsumer->consumerId); return 0; } static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { - mTrace("consumer:%ld, perform update action", pOldConsumer->consumerId); + mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId); // TODO handle update /*taosWLockLatch(&pOldConsumer->lock);*/ diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c6b93ad7b164ce0b6a7b18820991b65a78248c11..ead56c1f8cffbf4eafc0408d7f37cad4d36c3654 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -31,14 +31,14 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew); -static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq); -static int32_t mndProcessDropDbReq(SMnodeMsg *pReq); -static int32_t mndProcessUseDbReq(SMnodeMsg *pReq); -static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq); -static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq); -static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateDbReq(SNodeMsg *pReq); +static int32_t mndProcessAlterDbReq(SNodeMsg *pReq); +static int32_t mndProcessDropDbReq(SNodeMsg *pReq); +static int32_t mndProcessUseDbReq(SNodeMsg *pReq); +static int32_t mndProcessSyncDbReq(SNodeMsg *pReq); +static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); +static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); int32_t mndInitDb(SMnode *pMnode) { @@ -384,7 +384,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { +static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; memcpy(dbObj.name, pCreate->db, TSDB_DB_FNAME_LEN); memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); @@ -458,8 +458,8 @@ CREATE_DB_OVER: return code; } -static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -622,7 +622,7 @@ static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndUpdateDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); if (pTrans == NULL) goto UPDATE_DB_OVER; @@ -642,8 +642,8 @@ UPDATE_DB_OVER: return code; } -static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -802,7 +802,7 @@ static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { +static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_DB, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_DB_OVER; @@ -837,8 +837,8 @@ DROP_DB_OVER: return code; } -static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -952,8 +952,8 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { sdbCancelFetch(pSdb, pIter); } -static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -1039,8 +1039,8 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; USE_DB_OVER: if (code != 0) { @@ -1123,8 +1123,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } -static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessSyncDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -1164,8 +1164,8 @@ SYNC_DB_OVER: return code; } -static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCompactDbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -1205,8 +1205,8 @@ SYNC_DB_OVER: return code; } -static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -1314,11 +1314,11 @@ static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMe pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pShow->bytes[cols] = 1; - pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; - strcpy(pSchema[cols].name, "update"); - pSchema[cols].bytes = pShow->bytes[cols]; - cols++; +// pShow->bytes[cols] = 1; +// pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; +// strcpy(pSchema[cols].name, "update"); +// pSchema[cols].bytes = pShow->bytes[cols]; +// cols++; pMeta->numOfColumns = cols; pShow->numOfColumns = cols; @@ -1463,8 +1463,8 @@ static void setInformationSchemaDbCfg(SDbObj* pDbObj) { pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; } -static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rowsCapacity) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rowsCapacity) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SDbObj *pDb = NULL; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6e8d9aa79f8f9dfd1cd0e1a2689f91905600a2bc..f81dead3259c3e43d0287829dfcd33be6ba2fad1 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -27,6 +27,22 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + // TODO encode tasks + if (pObj->tasks) { + int32_t sz = taosArrayGetSize(pObj->tasks); + tEncodeI32(pEncoder, sz); + for (int32_t i = 0; i < sz; i++) { + SArray *pArray = taosArrayGet(pObj->tasks, i); + int32_t innerSz = taosArrayGetSize(pArray); + tEncodeI32(pEncoder, innerSz); + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask *pTask = taosArrayGet(pArray, j); + tEncodeSStreamTask(pEncoder, pTask); + } + } + } else { + tEncodeI32(pEncoder, 0); + } return pEncoder->pos; } @@ -39,8 +55,26 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1; if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeCStr(pDecoder, (const char **)&pObj->sql) < 0) return -1; - if (tDecodeCStr(pDecoder, (const char **)&pObj->logicalPlan) < 0) return -1; - if (tDecodeCStr(pDecoder, (const char **)&pObj->physicalPlan) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + if (sz != 0) { + pObj->tasks = taosArrayInit(sz, sizeof(SArray)); + for (int32_t i = 0; i < sz; i++) { + int32_t innerSz; + if (tDecodeI32(pDecoder, &innerSz) < 0) return -1; + SArray *pArray = taosArrayInit(innerSz, sizeof(SStreamTask)); + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask task; + if (tDecodeSStreamTask(pDecoder, &task) < 0) return -1; + taosArrayPush(pArray, &task); + } + taosArrayPush(pObj->tasks, pArray); + } + } else { + pObj->tasks = NULL; + } return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c274f65b24f119b6b31ea2780fab02643ac1df53..19dab72ae8d64d23b95ea18255e08b94a1f56fed 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -49,17 +49,17 @@ static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew); -static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessStatusReq(SMnodeMsg *pReq); - -static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq); +static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessStatusReq(SNodeMsg *pReq); + +static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); -static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetDnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); int32_t mndInitDnode(SMnode *pMnode) { @@ -296,8 +296,8 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return 0; } -static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessStatusReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SStatusReq statusReq = {0}; SDnodeObj *pDnode = NULL; int32_t code = -1; @@ -424,8 +424,8 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { tSerializeSStatusRsp(pHead, contLen, &statusRsp); taosArrayDestroy(statusRsp.pDnodeEps); - pReq->contLen = contLen; - pReq->pCont = pHead; + pReq->rspLen = contLen; + pReq->pRsp = pHead; } pDnode->lastAccessTime = curMs; @@ -437,7 +437,7 @@ PROCESS_STATUS_MSG_OVER: return code; } -static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pReq, SCreateDnodeReq *pCreate) { +static int32_t mndCreateDnode(SMnode *pMnode, SNodeMsg *pReq, SCreateDnodeReq *pCreate) { SDnodeObj dnodeObj = {0}; dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE); dnodeObj.createdTime = taosGetTimestampMs(); @@ -471,8 +471,8 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pReq, SCreateDnodeReq * return 0; } -static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDnodeObj *pDnode = NULL; @@ -521,7 +521,7 @@ CREATE_DNODE_OVER: return code; } -static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) { +static int32_t mndDropDnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_DNODE, &pReq->rpcMsg); if (pTrans == NULL) { mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); @@ -547,8 +547,8 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) return 0; } -static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDnodeObj *pDnode = NULL; @@ -596,8 +596,8 @@ DROP_DNODE_OVER: return code; } -static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SMCfgDnodeReq cfgReq = {0}; if (tDeserializeSMCfgDnodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { @@ -628,11 +628,11 @@ static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { return 0; } -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) { mInfo("app:%p config rsp from dnode", pRsp->rpcMsg.ahandle); } -static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { +static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchemas; @@ -663,8 +663,8 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t totalRows = 0; int32_t numOfRows = 0; char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; @@ -709,8 +709,8 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {} -static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetDnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -773,8 +773,8 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b2daf848c53262f45fc18196398ba84ff4fe1d38..b43d887dd79f9ad164d40722c8677474e155dcac 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -29,13 +29,13 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew); -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate); -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc); -static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq); -static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq); -static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq); -static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCreate); +static int32_t mndDropFunc(SMnode *pMnode, SNodeMsg *pReq, SFuncObj *pFunc); +static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq); +static int32_t mndProcessDropFuncReq(SNodeMsg *pReq); +static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq); +static int32_t mndGetFuncMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); int32_t mndInitFunc(SMnode *pMnode) { @@ -181,7 +181,7 @@ static void mndReleaseFunc(SMnode *pMnode, SFuncObj *pFunc) { sdbRelease(pSdb, pFunc); } -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate) { +static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCreate) { int32_t code = -1; STrans *pTrans = NULL; @@ -234,7 +234,7 @@ CREATE_FUNC_OVER: return code; } -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc) { +static int32_t mndDropFunc(SMnode *pMnode, SNodeMsg *pReq, SFuncObj *pFunc) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_FUNC, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_FUNC_OVER; @@ -262,8 +262,8 @@ DROP_FUNC_OVER: return code; } -static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SFuncObj *pFunc = NULL; @@ -339,8 +339,8 @@ CREATE_FUNC_OVER: return code; } -static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SFuncObj *pFunc = NULL; @@ -394,8 +394,8 @@ DROP_FUNC_OVER: return code; } -static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncRsp retrieveRsp = {0}; @@ -451,8 +451,8 @@ static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { tSerializeSRetrieveFuncRsp(pRsp, contLen, &retrieveRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; code = 0; @@ -463,8 +463,8 @@ RETRIEVE_FUNC_OVER: return code; } -static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetFuncMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -545,8 +545,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le return tDataTypes[type].name; } -static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SFuncObj *pFunc = NULL; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 01627f5557738cde36988fa7c02da05da25d2b21..703f36bb335248a8ff4db645210643529b6553cc 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -30,13 +30,13 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw); static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj); static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj); static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew); -static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveMnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateMnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropMnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateMnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessAlterMnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropMnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetMnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); int32_t mndInitMnode(SMnode *pMnode) { @@ -320,7 +320,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_ALTER_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -345,7 +345,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); return -1; @@ -355,7 +355,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno return 0; } -static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { +static int32_t mndCreateMnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { int32_t code = -1; SMnodeObj mnodeObj = {0}; @@ -380,8 +380,8 @@ CREATE_MNODE_OVER: return code; } -static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateMnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SMnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -490,7 +490,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_ALTER_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -517,7 +517,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_MNODE; - action.acceptableCode = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); return -1; @@ -527,7 +527,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode return 0; } -static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pReq, SMnodeObj *pObj) { +static int32_t mndDropMnode(SMnode *pMnode, SNodeMsg *pReq, SMnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_MNODE, &pReq->rpcMsg); @@ -547,8 +547,8 @@ DROP_MNODE_OVER: return code; } -static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropMnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SMnodeObj *pObj = NULL; @@ -595,23 +595,23 @@ DROP_MNODE_OVER: return code; } -static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessAlterMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropMnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetMnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -663,8 +663,8 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveMnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index ac9e99ebd43769559c47868275d84da2c4066001..38157cf220b6dfccb740fa1264c343f9625eb6a8 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -32,7 +32,7 @@ static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset); static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset); static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset); -static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq); +static int32_t mndProcessCommitOffsetReq(SNodeMsg *pReq); int32_t mndInitOffset(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_OFFSET, @@ -152,10 +152,10 @@ int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicNam return 0; } -static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) { +static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) { char key[TSDB_PARTITION_KEY_LEN]; - SMnode *pMnode = pMsg->pMnode; + SMnode *pMnode = pMsg->pNode; char *msgStr = pMsg->rpcMsg.pCont; SMqCMCommitOffsetReq commitOffsetReq; SCoder decoder; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index dbf299e8da04d30845985414642d6509d3dbd218..7ca1984457daef36d1b7bf4a3dee081a67d65a9e 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -50,14 +50,14 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); -static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); -static int32_t mndProcessConnectReq(SMnodeMsg *pReq); -static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq); -static int32_t mndProcessKillConnReq(SMnodeMsg *pReq); -static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); -static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq); +static int32_t mndProcessConnectReq(SNodeMsg *pReq); +static int32_t mndProcessKillQueryReq(SNodeMsg *pReq); +static int32_t mndProcessKillConnReq(SNodeMsg *pReq); +static int32_t mndGetConnsMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetQueryMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { @@ -177,8 +177,8 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { } } -static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessConnectReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SUserObj *pUser = NULL; SDbObj *pDb = NULL; SConnObj *pConn = NULL; @@ -206,8 +206,9 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { } if (connReq.db[0]) { - snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); - pDb = mndAcquireDb(pMnode, pReq->db); + char db[TSDB_DB_FNAME_LEN]; + snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); + pDb = mndAcquireDb(pMnode, db); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, connReq.db, terrstr()); @@ -237,8 +238,8 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { if (pRsp == NULL) goto CONN_OVER; tSerializeSConnectRsp(pRsp, contLen, &connectRsp); - pReq->contLen = contLen; - pReq->pCont = pRsp; + pReq->rspLen = contLen; + pReq->pRsp = pRsp; mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, connReq.app); @@ -338,8 +339,8 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { return NULL; } -static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SClientHbBatchReq batchReq = {0}; if (tDeserializeSClientHbBatchReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &batchReq) != 0) { @@ -423,12 +424,12 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } taosArrayDestroy(batchRsp.rsps); - pReq->contLen = tlen; - pReq->pCont = buf; + pReq->rspLen = tlen; + pReq->pRsp = buf; return 0; #if 0 - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SHeartBeatReq *pHeartbeat = pReq->rpcMsg.pCont; @@ -495,13 +496,13 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { mndReleaseConn(pMnode, pConn); pReq->contLen = sizeof(SConnectRsp); - pReq->pCont = pRsp; + pReq->pRsp = pRsp; return 0; #endif } -static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -534,8 +535,8 @@ static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { } } -static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillConnReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -566,8 +567,8 @@ static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { } } -static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetConnsMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -641,8 +642,8 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; @@ -700,8 +701,8 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in return numOfRows; } -static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetQueryMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -815,8 +816,8 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 980e1a5a3fb35ccf4ab14fd4f46dc662db052615..3c66160c916735515caa411b0712951db34f12e2 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -29,14 +29,14 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw); static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew); -static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateQnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateQnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetQnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); -static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq); +static int32_t mndProcessQnodeListReq(SNodeMsg *pReq); int32_t mndInitQnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_QNODE, @@ -205,7 +205,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -232,7 +232,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -242,7 +242,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } -static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { +static int32_t mndCreateQnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { int32_t code = -1; SQnodeObj qnodeObj = {0}; @@ -268,8 +268,8 @@ CREATE_QNODE_OVER: return code; } -static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateQnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SQnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -355,7 +355,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -365,7 +365,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn return 0; } -static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pReq, SQnodeObj *pObj) { +static int32_t mndDropQnode(SMnode *pMnode, SNodeMsg *pReq, SQnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_QNODE, &pReq->rpcMsg); @@ -384,8 +384,8 @@ DROP_QNODE_OVER: return code; } -static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SQnodeObj *pObj = NULL; @@ -432,12 +432,11 @@ DROP_QNODE_OVER: return code; } - -static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq) { +static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { int32_t code = -1; SQnodeListReq qlistReq = {0}; int32_t numOfRows = 0; - SMnode *pMnode = pReq->pMnode; + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; SQnodeObj *pObj = NULL; SQnodeListRsp qlistRsp = {0}; @@ -483,8 +482,8 @@ static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq) { tSerializeSQnodeListRsp(pRsp, rspLen, &qlistRsp); - pReq->contLen = rspLen; - pReq->pCont = pRsp; + pReq->rspLen = rspLen; + pReq->pRsp = pRsp; code = 0; QNODE_LIST_OVER: @@ -494,19 +493,18 @@ QNODE_LIST_OVER: return code; } - -static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateQnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetQnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -545,8 +543,8 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index e827810cc9850f9007a1c6b037fc85f9b0b5fa30..faef757e7625b14d69c395284c5e7a5921711c97 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -21,6 +21,7 @@ #include "mndOffset.h" #include "mndShow.h" #include "mndStb.h" +#include "mndStream.h" #include "mndSubscribe.h" #include "mndTopic.h" #include "mndTrans.h" @@ -28,10 +29,181 @@ #include "mndVgroup.h" #include "tcompare.h" #include "tname.h" +#include "tuuid.h" + +int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { + SSdb* pSdb = pMnode->pSdb; + SVgObj* pVgroup = NULL; + SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); + if (pPlan == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + ASSERT(pStream->vgNum == 0); + + int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); + pStream->tasks = taosArrayInit(totLevel, sizeof(SArray)); + + int32_t msgLen; + for (int32_t level = 0; level < totLevel; level++) { + SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask)); + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level); + int32_t opNum = LIST_LENGTH(inner->pNodeList); + ASSERT(opNum == 1); + + SSubplan* plan = nodesListGetNode(inner->pNodeList, level); + if (level == 0) { + ASSERT(plan->type == SUBPLAN_TYPE_SCAN); + void* pIter = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pStream->dbUid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + pStream->vgNum++; + // send to vnode + + SStreamTask* pTask = streamTaskNew(pStream->uid, level); + + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + taosArrayPush(taskOneLevel, pTask); + + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + int32_t tlen = sizeof(SMsgHead) + encoder.pos; + tCoderClear(&encoder); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + tCoderClear(&encoder); + + STransAction action = {0}; + action.epSet = plan->execNode.epSet; + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_VND_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + rpcFreeCont(buf); + return -1; + } + } + } else if (plan->subplanType == SUBPLAN_TYPE_SCAN) { + // duplicatable + + int32_t parallel = 0; + // if no snode, parallel set to fetch thread num in vnode + + // if has snode, set to shared thread num in snode + parallel = SND_SHARED_THREAD_NUM; + + for (int32_t i = 0; i < parallel; i++) { + SStreamTask* pTask = streamTaskNew(pStream->uid, level); + + // TODO:get snode id and ep + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + + taosArrayPush(taskOneLevel, pTask); + + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + int32_t tlen = sizeof(SMsgHead) + encoder.pos; + tCoderClear(&encoder); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + tCoderClear(&encoder); + + STransAction action = {0}; + action.epSet = plan->execNode.epSet; + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_SND_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + rpcFreeCont(buf); + return -1; + } + } + } else { + // not duplicatable + SStreamTask* pTask = streamTaskNew(pStream->uid, level); + + // TODO:get snode id and ep + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + taosArrayPush(taskOneLevel, pTask); + + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + int32_t tlen = sizeof(SMsgHead) + encoder.pos; + tCoderClear(&encoder); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + tCoderClear(&encoder); + + STransAction action = {0}; + action.epSet = plan->execNode.epSet; + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_SND_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + rpcFreeCont(buf); + return -1; + } + } + taosArrayPush(pStream->tasks, taskOneLevel); + } + return 0; +} int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { - SSdb* pSdb = pMnode->pSdb; - SVgObj* pVgroup = NULL; + SSdb* pSdb = pMnode->pSdb; + SVgObj* pVgroup = NULL; SQueryPlan* pPlan = qStringToQueryPlan(pTopic->physicalPlan); if (pPlan == NULL) { terrno = TSDB_CODE_QRY_INVALID_INPUT; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index ae2cf23b29fc982b4ee058587195964a73e7a23e..8536ee89814972532a28737c4da0490a23c890c2 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -22,10 +22,10 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); -static int32_t mndProcessShowReq(SMnodeMsg *pReq); -static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq); +static int32_t mndProcessShowReq(SNodeMsg *pReq); +static int32_t mndProcessRetrieveReq(SNodeMsg *pReq); static bool mndCheckRetrieveFinished(SShowObj *pShow); -static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq); +static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq); int32_t mndInitShow(SMnode *pMnode) { SShowMgmt *pMgmt = &pMnode->showMgmt; @@ -117,8 +117,8 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { taosCacheRelease(pMgmt->cache, (void **)(&pShow), forceRemove); } -static int32_t mndProcessShowReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessShowReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t code = -1; SShowReq showReq = {0}; @@ -161,8 +161,8 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) { int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp); void *pBuf = rpcMallocCont(bufLen); tSerializeSShowRsp(pBuf, bufLen, &showRsp); - pReq->contLen = bufLen; - pReq->pCont = pBuf; + pReq->rspLen = bufLen; + pReq->pRsp = pBuf; mndReleaseShowObj(pShow, false); } else { mndReleaseShowObj(pShow, true); @@ -178,8 +178,8 @@ SHOW_OVER: return code; } -static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; @@ -248,8 +248,8 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision - pReq->pCont = pRsp; - pReq->contLen = size; + pReq->pRsp = pRsp; + pReq->rspLen = size; if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; @@ -263,8 +263,8 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; @@ -364,8 +364,8 @@ static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) { pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision - pReq->pCont = pRsp; - pReq->contLen = size; + pReq->pRsp = pRsp; + pReq->rspLen = size; if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 6040aa088c8debc93a6149866e28da363762838c..dda02fecf21a8216306c7cef2c591b4c387ba3f0 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -29,12 +29,12 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw); static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew); -static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq); -static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateSnodeReq(SNodeMsg *pReq); +static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq); +static int32_t mndProcessCreateSnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp); +static int32_t mndGetSnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); int32_t mndInitSnode(SMnode *pMnode) { @@ -203,7 +203,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -230,7 +230,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -240,7 +240,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } -static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { +static int32_t mndCreateSnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { int32_t code = -1; SSnodeObj snodeObj = {0}; @@ -267,8 +267,8 @@ CREATE_SNODE_OVER: return code; } -static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateSnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SSnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; @@ -355,7 +355,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -365,7 +365,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn return 0; } -static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pReq, SSnodeObj *pObj) { +static int32_t mndDropSnode(SMnode *pMnode, SNodeMsg *pReq, SSnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_SNODE, &pReq->rpcMsg); @@ -385,8 +385,8 @@ DROP_SNODE_OVER: return code; } -static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SSnodeObj *pObj = NULL; @@ -433,18 +433,18 @@ DROP_SNODE_OVER: return code; } -static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateSnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetSnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -483,8 +483,8 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 35d1cf4515c4fda76bbb003b2eca3481109a4a0b..8929709b1c47b7b811dd958c39940d8ce8a974c2 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -33,15 +33,15 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); -static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq); -static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq); -static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq); -static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq); -static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq); +static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq); +static int32_t mndProcessMDropStbReq(SNodeMsg *pReq); +static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessVAlterStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessVDropStbRsp(SNodeMsg *pRsp); +static int32_t mndProcessTableMetaReq(SNodeMsg *pReq); +static int32_t mndGetStbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); int32_t mndInitStb(SMnode *pMnode) { @@ -490,7 +490,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); @@ -551,8 +551,8 @@ CREATE_STB_OVER: return code; } -static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SStbObj *pTopicStb = NULL; SStbObj *pStb = NULL; @@ -623,7 +623,7 @@ CREATE_STB_OVER: return code; } -static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -980,7 +980,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndAlterStb(SMnode *pMnode, SMnodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { +static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); @@ -1043,8 +1043,8 @@ ALTER_STB_OVER: return code; } -static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SDbObj *pDb = NULL; SStbObj *pStb = NULL; @@ -1096,7 +1096,7 @@ ALTER_STB_OVER: return code; } -static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVAlterStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -1160,7 +1160,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * return 0; } -static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { +static int32_t mndDropStb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_STB, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_STB_OVER; @@ -1180,8 +1180,8 @@ DROP_STB_OVER: return code; } -static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessMDropStbReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SDbObj *pDb = NULL; @@ -1237,7 +1237,7 @@ DROP_STB_OVER: return code; } -static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessVDropStbRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -1311,8 +1311,8 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } -static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessTableMetaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; STableInfoReq infoReq = {0}; STableMetaRsp metaRsp = {0}; @@ -1347,8 +1347,8 @@ static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); - pReq->pCont = pRsp; - pReq->contLen = rspLen; + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; code = 0; mDebug("stb:%s.%s, meta is retrieved", infoReq.dbFName, infoReq.tbName); @@ -1436,8 +1436,8 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs return 0; } -static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetStbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfStbs(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -1499,8 +1499,8 @@ static void mndExtractTableName(char *tableId, char *name) { } } -static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStbObj *pStb = NULL; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 54ad9cd7e239b20a84eaa6f7c791b1942413598f..45d45bef2ddbbda877f7258253e6e171205806a7 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -18,6 +18,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndScheduler.h" #include "mndShow.h" #include "mndStb.h" #include "mndTrans.h" @@ -31,12 +32,12 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); -static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq); -/*static int32_t mndProcessDropStreamReq(SMnodeMsg *pReq);*/ -/*static int32_t mndProcessDropStreamInRsp(SMnodeMsg *pRsp);*/ -static int32_t mndProcessStreamMetaReq(SMnodeMsg *pReq); -static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq); +/*static int32_t mndProcessDropStreamReq(SNodeMsg *pReq);*/ +/*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/ +static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq); +static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitStream(SMnode *pMnode) { @@ -208,7 +209,7 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { return 0; } -static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { mDebug("stream:%s to create", pCreate->name); SStreamObj streamObj = {0}; tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); @@ -229,6 +230,12 @@ static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamR } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); + if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { + mError("stream:%ld, schedule stream since %s", streamObj.uid, terrstr()); + mndTransDrop(pTrans); + return -1; + } + SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); @@ -247,8 +254,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamR return 0; } -static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SStreamObj *pStream = NULL; SDbObj *pDb = NULL; @@ -339,8 +346,8 @@ static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfS return 0; } -static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfStreams(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -383,8 +390,8 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index fa4fed59d09732462a67df20bead3324f23f8299..25ec5f7cd470700f140d331f5b60ac362a11fcf1 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -48,14 +48,14 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub); -static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg); -static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); -static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); -static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg); -static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg); +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); +static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg); +static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg); +static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg); static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, const SMqConsumerEp *pConsumerEp); @@ -211,8 +211,8 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq } #if 0 -static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; uint8_t *str = pMsg->rpcMsg.pCont; SMqCMResetOffsetReq req; @@ -249,14 +249,14 @@ static int32_t mndProcessResetOffsetReq(SMnodeMsg *pMsg) { } #endif -static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; SMqCMGetSubEpRsp rsp = {0}; int64_t consumerId = be64toh(pReq->consumerId); int32_t epoch = ntohl(pReq->epoch); - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); if (pConsumer == NULL) { terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; @@ -327,8 +327,8 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); tDeleteSMqCMGetSubEpRsp(&rsp); mndReleaseConsumer(pMnode, pConsumer); - pMsg->pCont = buf; - pMsg->contLen = tlen; + pMsg->pRsp = buf; + pMsg->rspLen = tlen; return 0; } @@ -356,8 +356,8 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebSub; } -static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer; void *pIter = NULL; @@ -420,7 +420,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg), }; - pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); + (*pMnode->putToWriteQFp)(pMnode->pWrapper, &rpcMsg); } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); @@ -428,8 +428,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg); void *pIter = NULL; @@ -448,7 +448,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) { int64_t lostConsumerId = *(int64_t *)taosArrayGet(pRebSub->lostConsumers, i); - mInfo("mq remove lost consumer %ld", lostConsumerId); + mInfo("mq remove lost consumer %" PRId64 "", lostConsumerId); for (int32_t j = 0; j < taosArrayGetSize(pSub->consumers); j++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); @@ -479,7 +479,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { else vgThisConsumerAfterRb = vgEachConsumer; - mInfo("mq consumer:%ld, connectted vgroup number change from %d to %d", pSubConsumer->consumerId, + mInfo("mq consumer:%" PRId64 ", connectted vgroup number change from %d to %d", pSubConsumer->consumerId, vgThisConsumerBeforeRb, vgThisConsumerAfterRb); while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) { @@ -503,7 +503,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); } - mInfo("mq consumer:%ld, status change from %d to %d", pRebConsumer->consumerId, status, pRebConsumer->status); + mInfo("mq consumer:%" PRId64 ", status change from %d to %d", pRebConsumer->consumerId, status, pRebConsumer->status); SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); @@ -537,13 +537,13 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { mndSplitSubscribeKey(pSub->key, topic, cgroup); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - mInfo("mq set conn: assign vgroup %d of topic %s to consumer %ld", pConsumerEp->vgId, topic, + mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 "", pConsumerEp->vgId, topic, pConsumerEp->consumerId); mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); mndReleaseTopic(pMnode, pTopic); } else { - mInfo("mq rebalance: assign vgroup %d, from consumer %ld to consumer %ld", pConsumerEp->vgId, + mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "", pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId); mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); @@ -994,8 +994,8 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } -static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; char *msgStr = pMsg->rpcMsg.pCont; SCMSubscribeReq subscribe; tDeserializeSCMSubscribeReq(msgStr, &subscribe); @@ -1099,7 +1099,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, newTopicName); bool createSub = false; if (pSub == NULL) { - mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName); + mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup, newTopicName); pSub = mndCreateSubscription(pMnode, pTopic, cgroup); createSub = true; @@ -1118,7 +1118,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { pConsumerEp->consumerId = consumerId; taosArrayPush(mqSubConsumer.vgInfo, pConsumerEp); if (pConsumerEp->oldConsumerId == -1) { - mInfo("mq set conn: assign vgroup %d of topic %s to consumer %ld", pConsumerEp->vgId, newTopicName, + mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 "", pConsumerEp->vgId, newTopicName, pConsumerEp->consumerId); mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); } else { @@ -1156,7 +1156,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 453535c6e762211fcd5d5ff9687197a2af9cd983..7993d7df9d7c4dbb5af38f97f7ded5a980acad05 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -79,8 +79,8 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { return pCont; } -static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { - SMnode* pMnode = pReq->pMnode; +static int32_t mndProcessTelemTimer(SNodeMsg* pReq) { + SMnode* pMnode = pReq->pNode; STelemMgmt* pMgmt = &pMnode->telemMgmt; if (!pMgmt->enable) return 0; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 7d7c5f9975f3498f34ffb54640dcbb59a0bfd891..e94ae4f8ecfdca0bc53b9b2238c672806d26a99c 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -31,12 +31,12 @@ static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic); -static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq); -static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq); -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp); -static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq); -static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq); +static int32_t mndProcessDropTopicReq(SNodeMsg *pReq); +static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp); +static int32_t mndProcessTopicMetaReq(SNodeMsg *pReq); +static int32_t mndGetTopicMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); int32_t mndInitTopic(SMnode *pMnode) { @@ -236,7 +236,30 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { return 0; } -static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { +static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) { + if (NULL == pCreate->ast) { + return TSDB_CODE_SUCCESS; + } + + SNode* pAst = NULL; + int32_t code = nodesStringToNode(pCreate->ast, &pAst); + + SQueryPlan* pPlan = NULL; + if (TSDB_CODE_SUCCESS == code) { + SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = true }; + code = qCreateQueryPlan(&cxt, &pPlan, NULL); + } + + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pPlan, false, pStr, NULL); + } + nodesDestroyNode(pAst); + nodesDestroyNode(pPlan); + terrno = code; + return code; +} + +static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); @@ -247,13 +270,23 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq topicObj.dbUid = pDb->uid; topicObj.version = 1; topicObj.sql = pCreate->sql; - topicObj.physicalPlan = pCreate->physicalPlan; - topicObj.logicalPlan = pCreate->logicalPlan; + topicObj.physicalPlan = ""; + topicObj.logicalPlan = ""; topicObj.sqlLen = strlen(pCreate->sql); + char* pPlanStr = NULL; + if (TSDB_CODE_SUCCESS != mndGetPlanString(pCreate, &pPlanStr)) { + mError("topic:%s, failed to get plan since %s", pCreate->name, terrstr()); + return -1; + } + if (NULL != pPlanStr) { + topicObj.physicalPlan = pPlanStr; + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + tfree(pPlanStr); return -1; } mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); @@ -261,6 +294,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + tfree(pPlanStr); mndTransDrop(pTrans); return -1; } @@ -268,16 +302,18 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + tfree(pPlanStr); mndTransDrop(pTrans); return -1; } + tfree(pPlanStr); mndTransDrop(pTrans); return 0; } -static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SMqTopicObj *pTopic = NULL; SDbObj *pDb = NULL; @@ -341,7 +377,7 @@ CREATE_TOPIC_OVER: return code; } -static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic) { +static int32_t mndDropTopic(SMnode *pMnode, SNodeMsg *pReq, SMqTopicObj *pTopic) { // TODO: cannot drop when subscribed STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { @@ -368,8 +404,8 @@ static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic return 0; } -static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropTopicReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SMDropTopicReq dropReq = {0}; if (tDeserializeSMDropTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { @@ -403,7 +439,7 @@ static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } @@ -435,8 +471,8 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo return 0; } -static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetTopicMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfTopics(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -479,8 +515,8 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqTopicObj *pTopic = NULL; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a009e01a52cfd1a535529cd71c47235b000a828a..d3b36aed31a39d592310a1d679e86b2f0f180b63 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -55,11 +55,11 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); static void mndTransExecute(SMnode *pMnode, STrans *pTrans); static void mndTransSendRpcRsp(STrans *pTrans); -static int32_t mndProcessTransReq(SMnodeMsg *pReq); -static int32_t mndProcessKillTransReq(SMnodeMsg *pReq); +static int32_t mndProcessTransReq(SNodeMsg *pReq); +static int32_t mndProcessKillTransReq(SNodeMsg *pReq); -static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveTrans(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetTransMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); int32_t mndInitTrans(SMnode *pMnode) { @@ -760,7 +760,7 @@ static void mndTransSendRpcRsp(STrans *pTrans) { } free(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, @@ -774,8 +774,8 @@ static void mndTransSendRpcRsp(STrans *pTrans) { } } -void mndTransProcessRsp(SMnodeMsg *pRsp) { - SMnode *pMnode = pRsp->pMnode; +void mndTransProcessRsp(SNodeMsg *pRsp) { + SMnode *pMnode = pRsp->pNode; int64_t signature = (int64_t)(pRsp->rpcMsg.ahandle); int32_t transId = (int32_t)(signature >> 32); int32_t action = (int32_t)((signature << 32) >> 32); @@ -816,7 +816,7 @@ void mndTransProcessRsp(SMnodeMsg *pRsp) { } } - mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%x", transId, action, pRsp->rpcMsg.code, + mDebug("trans:%d, action:%d response is received, code:0x%04x, accept:0x%04x", transId, action, pRsp->rpcMsg.code, pAction->acceptableCode); mndTransExecute(pMnode, pTrans); @@ -928,13 +928,13 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); return 0; } else { - mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode); + mError("trans:%d, all %d actions executed, code:0x%04x", pTrans->id, numOfActions, errCode & 0XFFFF); mndTransResetActions(pMnode, pTrans, pArray); terrno = errCode; return errCode; } } else { - mDebug("trans:%d, %d of %d actions executed, code:0x%x", pTrans->id, numOfReceived, numOfActions, errCode); + mDebug("trans:%d, %d of %d actions executed, code:0x%04x", pTrans->id, numOfReceived, numOfActions, errCode & 0XFFFF); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } } @@ -1111,7 +1111,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); } - mDebug("trans:%d, finished, code:0x%x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); + mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); return continueExec; } @@ -1157,8 +1157,8 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { mndTransSendRpcRsp(pTrans); } -static int32_t mndProcessTransReq(SMnodeMsg *pReq) { - mndTransPullup(pReq->pMnode); +static int32_t mndProcessTransReq(SNodeMsg *pReq) { + mndTransPullup(pReq->pNode); return 0; } @@ -1199,8 +1199,8 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { return 0; } -static int32_t mndProcessKillTransReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; SKillTransReq killReq = {0}; int32_t code = -1; SUserObj *pUser = NULL; @@ -1257,8 +1257,8 @@ void mndTransPullup(SMnode *pMnode) { sdbWriteFile(pMnode->pSdb); } -static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetTransMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -1320,8 +1320,8 @@ static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveTrans(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; STrans *pTrans = NULL; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b1cdf484b4805a9355da48db15cf5df5ca7ea91b..b7d32c01d938ca5522deefc79bf11629c16d103e 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -30,13 +30,13 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew); -static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SMnodeMsg *pReq); -static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq); -static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq); -static int32_t mndProcessDropUserReq(SMnodeMsg *pReq); -static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq); -static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SNodeMsg *pReq); +static int32_t mndProcessCreateUserReq(SNodeMsg *pReq); +static int32_t mndProcessAlterUserReq(SNodeMsg *pReq); +static int32_t mndProcessDropUserReq(SNodeMsg *pReq); +static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq); +static int32_t mndGetUserMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); int32_t mndInitUser(SMnode *pMnode) { @@ -261,7 +261,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { sdbRelease(pSdb, pUser); } -static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SMnodeMsg *pReq) { +static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate, SNodeMsg *pReq) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)pCreate->pass, strlen(pCreate->pass), userObj.pass); tstrncpy(userObj.user, pCreate->user, TSDB_USER_LEN); @@ -295,8 +295,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate return 0; } -static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -349,7 +349,7 @@ CREATE_USER_OVER: return code; } -static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SMnodeMsg *pReq) { +static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER,&pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to update since %s", pOld->user, terrstr()); @@ -397,8 +397,8 @@ static SHashObj *mndDupDbHash(SHashObj *pOld) { return pNew; } -static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -510,7 +510,7 @@ ALTER_USER_OVER: return code; } -static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pReq, SUserObj *pUser) { +static int32_t mndDropUser(SMnode *pMnode, SNodeMsg *pReq, SUserObj *pUser) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); @@ -536,8 +536,8 @@ static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pReq, SUserObj *pUser) { return 0; } -static int32_t mndProcessDropUserReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessDropUserReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SUserObj *pOperUser = NULL; @@ -585,8 +585,8 @@ DROP_USER_OVER: return code; } -static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; int32_t code = -1; SUserObj *pUser = NULL; SGetUserAuthReq authReq = {0}; @@ -635,8 +635,8 @@ static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); - pReq->pCont = pRsp; - pReq->contLen = contLen; + pReq->pRsp = pRsp; + pReq->rspLen = contLen; code = 0; GET_AUTH_OVER: @@ -647,8 +647,8 @@ GET_AUTH_OVER: return code; } -static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetUserMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -693,8 +693,8 @@ static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p return 0; } -static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SUserObj *pUser = NULL; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 4c268765e75f5f46f8254fa9d2ef20c0e9ecdf70..63d2385d4ff53a0b9d2ebf3573c514d697e8fa25 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -29,17 +29,17 @@ static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pRsp); - -static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessSyncVnodeRsp(SNodeMsg *pRsp); +static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); + +static int32_t mndGetVgroupMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetVnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { @@ -452,24 +452,24 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { return epset; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessCreateVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pRsp) { +static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pRsp) { return 0; } +static int32_t mndProcessSyncVnodeRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pRsp) { return 0; } +static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp) { return 0; } static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; @@ -500,8 +500,8 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetVgroupMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; if (mndGetVgroupMaxReplica(pMnode, pShow->db, &pShow->replica, &pShow->numOfRows) != 0) { @@ -551,8 +551,8 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp return 0; } -static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; @@ -654,8 +654,8 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndGetVnodeMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -694,8 +694,8 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * return 0; } -static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pMnode; +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index f2baea1cd9577b6a7592eac4edc976c4179d8d2c..39f2155109f164e3897df5ca2e425d2a2f8de1b2 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -44,27 +44,21 @@ #define TELEM_TIMER_MS 86400000 int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { - if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { + if (pMnode == NULL || pMnode->sendReqFp == NULL) { terrno = TSDB_CODE_MND_NOT_READY; return -1; } - return (*pMnode->sendReqToDnodeFp)(pMnode->pDnode, pEpSet, pMsg); + return (*pMnode->sendReqFp)(pMnode->pWrapper, pEpSet, pMsg); } int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg) { - if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { + if (pMnode == NULL || pMnode->sendReqFp == NULL) { terrno = TSDB_CODE_MND_NOT_READY; return -1; } - return (*pMnode->sendReqToMnodeFp)(pMnode->pDnode, pMsg); -} - -void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg) { - if (pMnode != NULL && pMnode->sendRedirectRspFp != NULL) { - (*pMnode->sendRedirectRspFp)(pMnode->pDnode, pMsg); - } + return (*pMnode->sendMnodeReqFp)(pMnode->pWrapper, pMsg); } static void *mndBuildTimerMsg(int32_t *pContLen) { @@ -86,7 +80,7 @@ static void mndPullupTrans(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; - pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); + pMnode->putToWriteQFp(pMnode->pWrapper, &rpcMsg); } taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); @@ -102,7 +96,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { .pCont = pReq, .contLen = contLen, }; - pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + pMnode->putToReadQFp(pMnode->pWrapper, &rpcMsg); } taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); @@ -114,7 +108,7 @@ static void mndPullupTelem(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; - pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + pMnode->putToReadQFp(pMnode->pWrapper, &rpcMsg); } taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); @@ -236,7 +230,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { } else { if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1; } - if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; + // if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; @@ -292,15 +286,14 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->replica = pOption->replica; pMnode->selfIndex = pOption->selfIndex; memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); - pMnode->pDnode = pOption->pDnode; - pMnode->putReqToMWriteQFp = pOption->putReqToMWriteQFp; - pMnode->putReqToMReadQFp = pOption->putReqToMReadQFp; - pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp; - pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp; - pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp; - - if (pMnode->sendReqToDnodeFp == NULL || pMnode->sendReqToMnodeFp == NULL || pMnode->sendRedirectRspFp == NULL || - pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { + pMnode->pWrapper = pOption->pWrapper; + pMnode->putToWriteQFp = pOption->putToWriteQFp; + pMnode->putToReadQFp = pOption->putToReadQFp; + pMnode->sendReqFp = pOption->sendReqFp; + pMnode->sendMnodeReqFp = pOption->sendMnodeReqFp; + + if (pMnode->sendReqFp == NULL || pMnode->sendMnodeReqFp == NULL || + pMnode->putToWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } @@ -386,108 +379,49 @@ int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption) { return 0; } -void mndDestroy(const char *path) { - mDebug("start to destroy mnode at %s", path); - taosRemoveDir(path); - mDebug("mnode is destroyed"); +int32_t mndStart(SMnode *pMnode) { + return mndInitTimer(pMnode); } -SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { - SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("failed to create msg since %s, app:%p RPC:%p", terrstr(), pRpcMsg->ahandle, pRpcMsg->handle); - return NULL; - } - - if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && - pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) { - SRpcConnInfo connInfo = {0}; - if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { - taosFreeQitem(pMsg); - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - mError("failed to create msg since %s, app:%p RPC:%p", terrstr(), pRpcMsg->ahandle, pRpcMsg->handle); - return NULL; - } - memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); - } +int32_t mndProcessMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SRpcMsg *pRpc = &pMsg->rpcMsg; + tmsg_t msgType = pMsg->rpcMsg.msgType; + void *ahandle = pMsg->rpcMsg.ahandle; + bool isReq = (pRpc->msgType & 1U); - pMsg->pMnode = pMnode; - pMsg->rpcMsg = *pRpcMsg; - pMsg->createdTime = taosGetTimestampSec(); - - if (pRpcMsg != NULL) { - mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); - } - return pMsg; -} - -void mndCleanupMsg(SMnodeMsg *pMsg) { - mTrace("msg:%p, is destroyed", pMsg); - rpcFreeCont(pMsg->rpcMsg.pCont); - pMsg->rpcMsg.pCont = NULL; - taosFreeQitem(pMsg); -} - -void mndSendRsp(SMnodeMsg *pMsg, int32_t code) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .code = code}; - rpcSendResponse(&rpcRsp); -} - -void mndProcessMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - int32_t code = 0; - tmsg_t msgType = pMsg->rpcMsg.msgType; - void *ahandle = pMsg->rpcMsg.ahandle; - bool isReq = (msgType & 1U); - - mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle); + mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(msgType), ahandle); if (isReq && !mndIsMaster(pMnode)) { - code = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_NOT_READY; mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; + return -1; } - if (isReq && pMsg->rpcMsg.pCont == NULL) { - code = TSDB_CODE_MND_INVALID_MSG_LEN; + if (isReq && (pRpc->contLen == 0 || pRpc->pCont == NULL)) { + terrno = TSDB_CODE_MND_INVALID_MSG_LEN; mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; + return -1; } MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(msgType)]; if (fp == NULL) { - code = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("msg:%p, failed to process since no msg handle, app:%p", pMsg, ahandle); - goto PROCESS_RPC_END; + return -1; } - code = (*fp)(pMsg); + int32_t code = (*fp)(pMsg); if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + terrno = code; mTrace("msg:%p, in progress, app:%p", pMsg, ahandle); - return; } else if (code != 0) { - code = terrno; mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - goto PROCESS_RPC_END; } else { mTrace("msg:%p, is processed, app:%p", pMsg, ahandle); } -PROCESS_RPC_END: - if (isReq) { - if (pMsg->rpcMsg.handle == NULL) return; - - if (code == TSDB_CODE_APP_NOT_READY) { - mndSendRedirectRsp(pMnode, &pMsg->rpcMsg); - } else if (code != 0) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont, .code = code}; - rpcSendResponse(&rpcRsp); - } else { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont}; - rpcSendResponse(&rpcRsp); - } - } + return code; } void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index d7d15df35a7a3838594ea6927bc84faef1457043..4d691692ffac460494fb9f07f141e3e68b292718 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -317,4 +317,4 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { ASSERT_NE(retry, retryMax); } -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 9dbc1be4e947cd41eff4e3aad255eda6ae31a68e..0282663b17a2ef0a8882e42c76408f76b0a83ef9 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -27,7 +27,7 @@ Testbase MndTestDb::test; TEST_F(MndTestDb, 01_ShowDb) { test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); - CHECK_META("show databases", 18); + CHECK_META("show databases", 17); CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); @@ -45,7 +45,7 @@ TEST_F(MndTestDb, 01_ShowDb) { CHECK_SCHEMA(14, TSDB_DATA_TYPE_TINYINT, 1, "comp"); CHECK_SCHEMA(15, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); CHECK_SCHEMA(16, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); - CHECK_SCHEMA(17, TSDB_DATA_TYPE_TINYINT, 1, "update"); +// CHECK_SCHEMA(17, TSDB_DATA_TYPE_TINYINT, 1, "update"); test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); @@ -85,7 +85,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { } test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); - CHECK_META("show databases", 18); + CHECK_META("show databases", 17); test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); @@ -173,7 +173,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { test.Restart(); test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); - CHECK_META("show databases", 18); + CHECK_META("show databases", 17); test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); @@ -215,7 +215,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { } test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); - CHECK_META("show databases", 18); + CHECK_META("show databases", 17); test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); @@ -255,7 +255,7 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { } test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); - CHECK_META("show databases", 18); + CHECK_META("show databases", 17); test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp index f58d0a67717bddc9a47cde4c3e3245a6ba70a096..73eefd875d28b48fb6e62601bba3dfaa857e8c79 100644 --- a/source/dnode/mnode/impl/test/topic/topic.cpp +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -65,8 +65,7 @@ void* MndTestTopic::BuildCreateTopicReq(const char* topicName, const char* sql, strcpy(createReq.name, topicName); createReq.igExists = 0; createReq.sql = (char*)sql; - createReq.physicalPlan = (char*)"physicalPlan"; - createReq.logicalPlan = (char*)"logicalPlan"; + createReq.ast = NULL; int32_t contLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index cea93017f4bbc1dd042d55a46a01561dfebd5881..9ddb3594bb92634895910032a7595b0866f2d427 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -204,6 +204,8 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } + taosMsleep(1000); + { // show trans test.SendShowMetaReq(TSDB_MGMT_TABLE_TRANS, ""); @@ -241,6 +243,7 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { EXPECT_EQ(test.GetShowRows(), 0); } + uInfo("======== re-create trans"); // re-create trans { SMCreateQnodeReq createReq = {0}; @@ -255,10 +258,14 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } + uInfo("======== kill and restart server") KillThenRestartServer(); + uInfo("======== server2 start") server2.DoStart(); + uInfo("======== server2 started") + { int32_t retry = 0; int32_t retryMax = 20; diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index a3fe4dfb14996f4e92d0a189f8f6ca07a2a6449e..2849da5c2ec1910c42fd62a95c4eb1629c56ad71 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -196,7 +196,7 @@ int32_t sdbReadFile(SSdb *pSdb) { } int32_t totalLen = sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t); - if (!taosCheckChecksumWhole((const uint8_t *)pRaw, totalLen) != 0) { + if ((!taosCheckChecksumWhole((const uint8_t *)pRaw, totalLen)) != 0) { code = TSDB_CODE_CHECKSUM_ERROR; mError("failed to read file:%s since %s", file, tstrerror(code)); break; diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index a1c3f5b0d42925d86d28a7e21dfd5dcb4cd8ae96..a257b343c26daa6cd6234ef9bf5d072263c6c4da 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -30,7 +30,7 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { } if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, pQnode, - (putReqToQueryQFp)qnodePutReqToVQueryQ, (sendReqToDnodeFp)qnodeSendReqToDnode)) { + (putReqToQueryQFp)qnodePutReqToVQueryQ, (sendReqFp)qnodeSendReqToDnode)) { tfree(pQnode); return NULL; } diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index dafd5d659416b04f027e85aa33122c6371e6d153..0c63e35e875ec28f61bfe9f6f9bd1f21f9300763 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -7,8 +7,10 @@ target_include_directories( ) target_link_libraries( snode + PRIVATE executor PRIVATE transport PRIVATE os PRIVATE common PRIVATE util -) \ No newline at end of file + PRIVATE qcom +) diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 5851e18478163f2a0370a00b1a71412467cffcc7..d1122fc4ecaae129ada2e427bb223cb6780e8e44 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -30,47 +30,34 @@ extern "C" { #endif enum { - STREAM_STATUS__READY = 1, + STREAM_STATUS__RUNNING = 1, STREAM_STATUS__STOPPED, STREAM_STATUS__CREATING, STREAM_STATUS__STOPING, - STREAM_STATUS__RESUMING, + STREAM_STATUS__RESTORING, STREAM_STATUS__DELETING, }; -enum { - STREAM_RUNNER__RUNNING = 1, - STREAM_RUNNER__STOP, -}; +typedef struct { + SHashObj* pHash; // taskId -> SStreamTask +} SStreamMeta; typedef struct SSnode { - SSnodeOpt cfg; + SStreamMeta* pMeta; + SSnodeOpt cfg; } SSnode; -typedef struct { - int64_t streamId; - int32_t IdxInLevel; - int32_t level; -} SStreamInfo; +SStreamMeta* sndMetaNew(); +void sndMetaDelete(SStreamMeta* pMeta); -typedef struct { - SStreamInfo meta; - int8_t status; - void* executor; - STaosQueue* queue; - void* stateStore; - // storage handle -} SStreamRunner; - -typedef struct { - SHashObj* pHash; -} SStreamMeta; +int32_t sndMetaDeployTask(SStreamMeta* pMeta, SStreamTask* pTask); +SStreamTask* sndMetaGetTask(SStreamMeta* pMeta, int32_t taskId); +int32_t sndMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); -int32_t sndCreateStream(); -int32_t sndDropStream(); +int32_t sndDropTaskOfStream(SStreamMeta* pMeta, int64_t streamId); -int32_t sndStopStream(); -int32_t sndResumeStream(); +int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId); +int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId); #ifdef __cplusplus } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 91008dd03ac8847b8a29d84b5324ece3cbb9c5c1..c0bfc2c3f5fc3d6e5a1f987e84ce45cb4e175ccb 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -13,33 +13,107 @@ * along with this program. If not, see . */ +#include "executor.h" #include "sndInt.h" +#include "tuuid.h" SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *pSnode = calloc(1, sizeof(SSnode)); + if (pSnode == NULL) { + return NULL; + } memcpy(&pSnode->cfg, pOption, sizeof(SSnodeOpt)); + pSnode->pMeta = sndMetaNew(); + if (pSnode->pMeta == NULL) { + free(pSnode); + return NULL; + } return pSnode; } -void sndClose(SSnode *pSnode) { free(pSnode); } +void sndClose(SSnode *pSnode) { + sndMetaDelete(pSnode->pMeta); + free(pSnode); +} int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } -int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - *pRsp = NULL; - return 0; +SStreamMeta *sndMetaNew() { + SStreamMeta *pMeta = calloc(1, sizeof(SStreamMeta)); + if (pMeta == NULL) { + return NULL; + } + pMeta->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMeta->pHash == NULL) { + free(pMeta); + return NULL; + } + return pMeta; +} + +void sndMetaDelete(SStreamMeta *pMeta) { + taosHashCleanup(pMeta->pHash); + free(pMeta); } -void sndDestroy(const char *path) {} +int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) { + pTask->executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL); + return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *)); +} + +SStreamTask *sndMetaGetTask(SStreamMeta *pMeta, int32_t taskId) { + return taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t)); +} -int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { - // stream deployment +int32_t sndMetaRemoveTask(SStreamMeta *pMeta, int32_t taskId) { + SStreamTask *pTask = taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t)); + if (pTask == NULL) { + return -1; + } + free(pTask->qmsg); + // TODO:free executor + free(pTask); + return taosHashRemove(pMeta->pHash, &taskId, sizeof(int32_t)); +} + +static int32_t sndProcessTaskExecReq(SSnode *pSnode, SRpcMsg *pMsg) { + SMsgHead *pHead = pMsg->pCont; + int32_t taskId = pHead->streamTaskId; + SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId); + if (pTask == NULL) { + return -1; + } + return 0; +} + +void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { + // stream deploy // stream stop/resume // operator exec - return 0; + if (pMsg->msgType == TDMT_SND_TASK_DEPLOY) { + void *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + SStreamTask *pTask = malloc(sizeof(SStreamTask)); + if (pTask == NULL) { + ASSERT(0); + } + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, msg, pMsg->contLen - sizeof(SMsgHead), TD_DECODER); + tDecodeSStreamTask(&decoder, pTask); + tCoderClear(&decoder); + + sndMetaDeployTask(pSnode->pMeta, pTask); + } else if (pMsg->msgType == TDMT_SND_TASK_EXEC) { + sndProcessTaskExecReq(pSnode, pMsg); + } else { + ASSERT(0); + } } -int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { +void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { // operator exec - return 0; + if (pMsg->msgType == TDMT_SND_TASK_EXEC) { + sndProcessTaskExecReq(pSnode, pMsg); + } else { + ASSERT(0); + } } diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index b4d45e83fb340202d39b5674dfbcf1d74b37179f..8d8ed2e427087798c7e6da03be47743cec526bfa 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -55,6 +55,7 @@ int tqCommit(STQ*); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); +int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); #ifdef __cplusplus } diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 87edfb8dde59deabdb28b8d6b03ae7bf1a67cb6c..7b93f7580d02465fe16ddf362aa558d17121e6a4 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -107,10 +107,8 @@ int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); // TODO: This is the basic params, and should wrap the params to a queryHandle. -int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, - int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, - int32_t nMaxResult); - +int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, + tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult); // STsdbCfg int tsdbOptionsInit(STsdbCfg *); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d51bf1b196d3b442589dce3bdbddde12cc60c955..776e4e47ac3378edfb9476936bd235a25acb4472 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -30,11 +30,8 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SVnode SVnode; -typedef struct SDnode SDnode; -typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); - +typedef struct SMgmtWrapper SMgmtWrapper; +typedef struct SVnode SVnode; typedef struct { // TODO int32_t reserved; @@ -43,7 +40,7 @@ typedef struct { typedef struct { int32_t vgId; uint64_t dbId; - SDnode *pDnode; + void *pWrapper; STfs *pTfs; uint64_t wsize; uint64_t ssize; @@ -63,9 +60,11 @@ typedef struct { } SVnodeCfg; typedef struct { - uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) - PutReqToVQueryQFp putReqToVQueryQFp; - SendReqToDnodeFp sendReqToDnodeFp; + uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) + PutToQueueFp putToQueryQFp; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SVnodeOpt; typedef struct { diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index fc6a3699d555774baa6366aad67b70653bc46411..9b21cf92ce83a161e56031cf8569f1337f3a180c 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -161,6 +161,7 @@ struct STQ { STqMemRef tqMemRef; STqMetaStore* tqMeta; STqPushMgr* tqPushMgr; + SHashObj* pStreamTasks; SWal* pWal; SMeta* pVnodeMeta; }; diff --git a/source/dnode/vnode/src/inc/tsdbDBDef.h b/source/dnode/vnode/src/inc/tsdbDBDef.h index 2e37b0ba45132a92e4de5b7e0a21e680564256bc..ca9b60049eeda2c6788e2675e9d6f2d69d2cf939 100644 --- a/source/dnode/vnode/src/inc/tsdbDBDef.h +++ b/source/dnode/vnode/src/inc/tsdbDBDef.h @@ -26,8 +26,9 @@ typedef struct SDBFile SDBFile; typedef DB_ENV* TDBEnv; struct SDBFile { - DB* pDB; - char* path; + int32_t fid; + DB* pDB; + char* path; }; int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF); diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 6f91b4d3abb4bf608c50ca15973f83b19fb1dbf6..f7fdc818d0c15f72cecac633e4799d07a26f78c5 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -16,6 +16,7 @@ #ifndef _TD_TSDB_DEF_H_ #define _TD_TSDB_DEF_H_ +#include "tsdbDBDef.h" #include "tmallocator.h" #include "meta.h" #include "tcompression.h" @@ -27,7 +28,6 @@ #include "ttime.h" #include "tsdb.h" -#include "tsdbDBDef.h" #include "tsdbCommit.h" #include "tsdbFS.h" #include "tsdbFile.h" @@ -46,7 +46,7 @@ extern "C" { struct STsdb { int32_t vgId; bool repoLocked; - pthread_mutex_t mutex; + TdThreadMutex mutex; char * path; STsdbCfg config; STsdbMemTable * mem; @@ -60,10 +60,12 @@ struct STsdb { SSmaEnv * pRSmaEnv; }; -#define REPO_ID(r) ((r)->vgId) -#define REPO_CFG(r) (&(r)->config) -#define REPO_FS(r) (r)->fs -#define IS_REPO_LOCKED(r) (r)->repoLocked +#define REPO_ID(r) ((r)->vgId) +#define REPO_CFG(r) (&(r)->config) +#define REPO_FS(r) (r)->fs +#define REPO_TFS(r) (r)->pTfs +#define IS_REPO_LOCKED(r) (r)->repoLocked +#define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv) int tsdbLockRepo(STsdb *pTsdb); int tsdbUnlockRepo(STsdb *pTsdb); diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index 96c5f8468f2ac97d047683220acbae6cf2e8dc8a..8156cbae00b19115f9bbad5bce8f2393e1daadcc 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -42,29 +42,29 @@ typedef struct { typedef struct { STsdbFSMeta meta; // FS meta SArray * df; // data file array - SArray * sf; // sma data file array v2(t|r)1900.index_name_1 + SArray * sf; // sma data file array v2f1900.index_name_1 } SFSStatus; /** * @brief Directory structure of .tsma data files. * - * /vnode2/tsdb $ tree .sma/ - * .sma/ - * ├── v2t100.index_name_1 - * ├── v2t101.index_name_1 - * ├── v2t102.index_name_1 - * ├── v2t1900.index_name_3 - * ├── v2t1901.index_name_3 - * ├── v2t1902.index_name_3 - * ├── v2t200.index_name_2 - * ├── v2t201.index_name_2 - * └── v2t202.index_name_2 + * /vnode2/tsdb $ tree tsma/ + * tsma/ + * ├── v2f100.index_name_1 + * ├── v2f101.index_name_1 + * ├── v2f102.index_name_1 + * ├── v2f1900.index_name_3 + * ├── v2f1901.index_name_3 + * ├── v2f1902.index_name_3 + * ├── v2f200.index_name_2 + * ├── v2f201.index_name_2 + * └── v2f202.index_name_2 * * 0 directories, 9 files */ typedef struct { - pthread_rwlock_t lock; + TdThreadRwlock lock; SFSStatus *cstatus; // current status SHashObj * metaCache; // meta cache @@ -108,7 +108,7 @@ SDFileSet *tsdbFSIterNext(SFSIter *pIter); int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_rdlock(&(pFs->lock)); + int code = taosThreadRwlockRdlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -117,7 +117,7 @@ static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { } static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_wrlock(&(pFs->lock)); + int code = taosThreadRwlockWrlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -126,7 +126,7 @@ static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { } static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { - int code = pthread_rwlock_unlock(&(pFs->lock)); + int code = taosThreadRwlockUnlock(&(pFs->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 649b5a2d47a102ee7fdf34b649609ffc7a08d4c2..f934b0263d36fd406cc4c80cad1b41f76e05c91a 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -20,13 +20,15 @@ typedef struct SSmaStat SSmaStat; typedef struct SSmaEnv SSmaEnv; struct SSmaEnv { - pthread_rwlock_t lock; - TDBEnv dbEnv; - char * path; + TdThreadRwlock lock; + SDiskID did; + TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? + char * path; // relative path SSmaStat * pStat; }; #define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_DID(env) ((env)->did) #define SMA_ENV_ENV(env) ((env)->dbEnv) #define SMA_ENV_PATH(env) ((env)->path) #define SMA_ENV_STAT(env) ((env)->pStat) @@ -49,7 +51,7 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId, } static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_rdlock(&(pEnv->lock)); + int code = taosThreadRwlockRdlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -58,7 +60,7 @@ static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { } static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_wrlock(&(pEnv->lock)); + int code = taosThreadRwlockWrlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -67,7 +69,7 @@ static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { } static FORCE_INLINE int tsdbUnLockSma(SSmaEnv *pEnv) { - int code = pthread_rwlock_unlock(&(pEnv->lock)); + int code = taosThreadRwlockUnlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 6f4f0049e3b728103122cbf71d9b16f37bd88c53..d3f0dce2e5765cad61fbe48a499d5f58953c89ec 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -48,14 +48,13 @@ typedef struct SVnodeMgr { // For commit bool stop; uint16_t nthreads; - pthread_t* threads; - pthread_mutex_t mutex; - pthread_cond_t hasTask; + TdThread* threads; + TdThreadMutex mutex; + TdThreadCond hasTask; TD_DLIST(SVnodeTask) queue; // For vnode Mgmt - SDnode* pDnode; - PutReqToVQueryQFp putReqToVQueryQFp; - SendReqToDnodeFp sendReqToDnodeFp; + PutToQueueFp putToQueryQFp; + SendReqFp sendReqFp; } SVnodeMgr; extern SVnodeMgr vnodeMgr; @@ -79,14 +78,14 @@ struct SVnode { SWal* pWal; tsem_t canCommit; SQHandle* pQuery; - SDnode* pDnode; + void* pWrapper; STfs* pTfs; }; int vnodeScheduleTask(SVnodeTask* task); -int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); -void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); +int32_t vnodePutToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); +void vnodeSendReq(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); #define vFatal(...) \ do { \ diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index b6a6c527da49cf3bdfa6c4cc8740acb9d4d97122..3eb9a480acff7f926b19f6796f0a326583fdbc4e 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -33,7 +33,7 @@ typedef struct { struct SMetaDB { #if IMPL_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif // DB DB *pTbDB; @@ -317,7 +317,7 @@ static SMetaDB *metaNewDB() { } #if IMPL_WITH_LOCK - pthread_rwlock_init(&pDB->rwlock, NULL); + taosThreadRwlockInit(&pDB->rwlock, NULL); #endif return pDB; @@ -326,7 +326,7 @@ static SMetaDB *metaNewDB() { static void metaFreeDB(SMetaDB *pDB) { if (pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_destroy(&pDB->rwlock); + taosThreadRwlockDestroy(&pDB->rwlock); #endif free(pDB); } @@ -636,7 +636,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { } // Decode - pCfg = (STSma *)malloc(sizeof(STSma)); + pCfg = (STSma *)calloc(1, sizeof(STSma)); if (pCfg == NULL) { return NULL; } @@ -896,7 +896,7 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) { STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { STSmaWrapper *pSW = NULL; - pSW = calloc(sizeof(*pSW), 1); + pSW = calloc(1, sizeof(*pSW)); if (pSW == NULL) { return NULL; } @@ -977,18 +977,18 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { static void metaDBWLock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_wrlock(&(pDB->rwlock)); + taosThreadRwlockWrlock(&(pDB->rwlock)); #endif } static void metaDBRLock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_rdlock(&(pDB->rwlock)); + taosThreadRwlockRdlock(&(pDB->rwlock)); #endif } static void metaDBULock(SMetaDB *pDB) { #if IMPL_WITH_LOCK - pthread_rwlock_unlock(&(pDB->rwlock)); + taosThreadRwlockUnlock(&(pDB->rwlock)); #endif } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d15481b4aa54c6c5009f0030eca63b0685a749ec..8cdc250e8d5bbb9cbaac311a38ac2001144572a1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -55,6 +55,8 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S return NULL; } + pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + return pTq; } @@ -416,3 +418,18 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { terrno = TSDB_CODE_SUCCESS; return 0; } + +int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { + SStreamTask* pTask = malloc(sizeof(SStreamTask)); + if (pTask == NULL) { + return -1; + } + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, (uint8_t*)msg, msgLen, TD_DECODER); + tDecodeSStreamTask(&decoder, pTask); + tCoderClear(&decoder); + + taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask)); + + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index a2342ec85aa066fab1bcbd2f7e93953151711508..0c4b933c198bf47aa9e8590afb186d66b28c2fc8 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tdatablock.h" #include "vnode.h" STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { @@ -128,10 +129,13 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int j = 0; for (int32_t i = 0; i < colNumNeed; i++) { - int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); + int16_t colId = *(int16_t*)taosArrayGet(pHandle->pColIdList, i); while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { j++; } + if (j >= pSchemaWrapper->nCols) { + continue; + } SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; SColumnInfoData colInfo = {0}; int sz = numOfRows * pColSchema->bytes; @@ -145,6 +149,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { taosArrayDestroy(pArray); return NULL; } + + blockDataEnsureColumnCapacity(&colInfo, numOfRows); taosArrayPush(pArray, &colInfo); } diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index cf3351c5d892c9c884a3a1050d9e9e92c3977251..ee279abf47ff6acfe406d9adff26bdf4dd9ed59b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -68,8 +68,8 @@ int32_t tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL, 0); if (ret != 0) { - // BDB_PERR("Failed to open tsdb env", ret); - tsdbWarn("Failed to open tsdb env for path %s since %d", path ? path : "NULL", ret); + terrno = TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR; + tsdbWarn("Failed to open tsdb env for path %s since ret %d != 0", path ? path : "NULL", ret); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index a03739c90feb9155040e28361f82cd3ce1c80e23..0f0d50ad085ad0bf92654830e24e594699cf5161 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -14,8 +14,8 @@ */ #include -#include "os.h" #include "tsdbDef.h" +#include "os.h" typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; @@ -203,7 +203,7 @@ STsdbFS *tsdbNewFS(const STsdbCfg *pCfg) { return NULL; } - int code = pthread_rwlock_init(&(pfs->lock), NULL); + int code = taosThreadRwlockInit(&(pfs->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); free(pfs); @@ -241,7 +241,7 @@ void *tsdbFreeFS(STsdbFS *pfs) { taosHashCleanup(pfs->metaCache); pfs->metaCache = NULL; pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); - pthread_rwlock_destroy(&(pfs->lock)); + taosThreadRwlockDestroy(&(pfs->lock)); free(pfs); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index b756cc9862efe60e496d3d43e995398af8719024..00e97c7b614b9277040a38f2487ea620d1c2777f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -23,8 +23,8 @@ static const char *TSDB_FNAME_SUFFIX[] = { "smal", // TSDB_FILE_SMAL "", // TSDB_FILE_MAX "meta", // TSDB_FILE_META - "sma", // TSDB_FILE_TSMA(directory name) - "sma", // TSDB_FILE_RSMA(directory name) + "tsma", // TSDB_FILE_TSMA + "rsma", // TSDB_FILE_RSMA }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index afa8921c004aa37282d86ff44f2d1f4850f2dfdc..131e92cab60aa1e26f5689ef5e61520e503c2d84 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -110,7 +110,7 @@ static void tsdbCloseImpl(STsdb *pTsdb) { } int tsdbLockRepo(STsdb *pTsdb) { - int code = pthread_mutex_lock(&pTsdb->mutex); + int code = taosThreadMutexLock(&pTsdb->mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -123,7 +123,7 @@ int tsdbLockRepo(STsdb *pTsdb) { int tsdbUnlockRepo(STsdb *pTsdb) { ASSERT(IS_REPO_LOCKED(pTsdb)); pTsdb->repoLocked = false; - int code = pthread_mutex_unlock(&pTsdb->mutex); + int code = taosThreadMutexUnlock(&pTsdb->mutex); if (code != 0) { tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -298,7 +298,7 @@ STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { } int tsdbLockRepo(STsdbRepo *pRepo) { - int code = pthread_mutex_lock(&pRepo->mutex); + int code = taosThreadMutexLock(&pRepo->mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -311,7 +311,7 @@ int tsdbLockRepo(STsdbRepo *pRepo) { int tsdbUnlockRepo(STsdbRepo *pRepo) { ASSERT(IS_REPO_LOCKED(pRepo)); pRepo->repoLocked = false; - int code = pthread_mutex_unlock(&pRepo->mutex); + int code = taosThreadMutexUnlock(&pRepo->mutex); if (code != 0) { tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -388,7 +388,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { tsdbError("vgId:%d no config changed", REPO_ID(repo)); } - int code = pthread_mutex_lock(&repo->save_mutex); + int code = taosThreadMutexLock(&repo->save_mutex); if (code != 0) { tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); @@ -416,7 +416,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { repo->config_changed = true; - pthread_mutex_unlock(&repo->save_mutex); + taosThreadMutexUnlock(&repo->save_mutex); // schedule a commit msg and wait for the new config applied tsdbSyncCommitConfig(repo); @@ -690,14 +690,14 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { pRepo->repoLocked = false; pRepo->pthread = NULL; - int code = pthread_mutex_init(&(pRepo->mutex), NULL); + int code = taosThreadMutexInit(&(pRepo->mutex), NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); tsdbFreeRepo(pRepo); return NULL; } - code = pthread_mutex_init(&(pRepo->save_mutex), NULL); + code = taosThreadMutexInit(&(pRepo->save_mutex), NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); tsdbFreeRepo(pRepo); @@ -747,7 +747,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); tsem_destroy(&(pRepo->readyToCommit)); - pthread_mutex_destroy(&pRepo->mutex); + taosThreadMutexDestroy(&pRepo->mutex); free(pRepo); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 01813af556715dd2e86b8203cd314f0428ffb327..e46d1a0ed47fc9843a6cce6f5c6f751762e717a5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -541,7 +541,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { } } if (addNew) { - int code = pthread_cond_signal(&pBufPool->poolNotEmpty); + int code = taosThreadCondSignal(&pBufPool->poolNotEmpty); if (code != 0) { if (tsdbUnlockRepo(pRepo) < 0) return -1; tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 66d50c1f9ec3f03e22a4e468dc017c558ff6f8ba..5af9e18ee1ebf0f49b11a4c32392bc42045d7bf1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tsdbDef.h" #include #include "os.h" #include "talgo.h" @@ -20,7 +21,6 @@ #include "tdataformat.h" #include "texception.h" #include "tsdb.h" -#include "tsdbDef.h" #include "tsdbFS.h" #include "tsdbLog.h" #include "tsdbReadImpl.h" @@ -3392,7 +3392,7 @@ void filterPrepare(void* expr, void* param) { if (size < (uint32_t)pSchema->bytes) { size = pSchema->bytes; } - // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space. + // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(TdUcs4) space. pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); tVariantDump(pCond, pInfo->q, pSchema->type, true); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 7335e4f58543d00ded7227d54a1be0606cf81de5..0eb2d525b3fb726890ccae0befa71a9db7c35890 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -15,9 +15,14 @@ #include "tsdbDef.h" -#undef SMA_PRINT_DEBUG_LOG +static const char *TSDB_SMA_DNAME[] = { + "", // TSDB_SMA_TYPE_BLOCK + "tsma", // TSDB_SMA_TYPE_TIME_RANGE + "rsma", // TSDB_SMA_TYPE_ROLLUP +}; +#undef _TEST_SMA_PRINT_DEBUG_LOG_ #define SMA_STORAGE_TSDB_DAYS 30 -#define SMA_STORAGE_TSDB_TIMES 30 +#define SMA_STORAGE_TSDB_TIMES 10 #define SMA_STORAGE_SPLIT_HOURS 24 #define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8 @@ -27,8 +32,8 @@ #define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test #define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test typedef enum { - SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2t200.dat - SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/rsma/sma_index_uid/v2r200.dat + SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma + SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma } ESmaStorageLevel; typedef struct { @@ -41,6 +46,7 @@ typedef struct { int32_t iter; int32_t fid; } SmaFsIter; + typedef struct { STsdb * pTsdb; SDBFile dFile; @@ -61,40 +67,55 @@ typedef struct { */ int8_t state; // ETsdbSmaStat SHashObj *expiredWindows; // key: skey of time window, value: N/A - STSma * pSma; + STSma * pSma; // cache schema } SSmaStatItem; struct SSmaStat { - SHashObj *smaStatItems; // key: indexName, value: SSmaStatItem + SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem + T_REF_DECLARE() }; // declaration of static functions -static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); -static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); -// TODO: This is the basic params, and should wrap the params to a queryHandle. -static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, - int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, - int32_t nMaxResult); -static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg); +// expired window +static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path); -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv); -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); -static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); -static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); -static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData); -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did); +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv); +static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey); +static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); +static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); + +// read data +// TODO: This is the basic params, and should wrap the params to a queryHandle. +static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, + int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, + int32_t nMaxResult); +// insert data +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); +static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); +static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); +static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); +static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData); +static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid); -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey); +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid); +static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); +static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); +static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); +static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { +// implementation +static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP, + TSDB_SMA_DNAME[smaType]); +} + +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) { SSmaEnv *pEnv = NULL; pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv)); @@ -103,7 +124,7 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return NULL; } - int code = pthread_rwlock_init(&(pEnv->lock), NULL); + int code = taosThreadRwlockInit(&(pEnv->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); free(pEnv); @@ -117,12 +138,16 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return NULL; } + pEnv->did = did; + if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) { tsdbFreeSmaEnv(pEnv); return NULL; } - if (tsdbOpenBDBEnv(&pEnv->dbEnv, pEnv->path) != TSDB_CODE_SUCCESS) { + char aname[TSDB_FILENAME_LEN] = {0}; + tfsAbsoluteName(pTsdb->pTfs, did, path, aname); + if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { tsdbFreeSmaEnv(pEnv); return NULL; } @@ -130,32 +155,18 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { return pEnv; } -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) { +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv) { if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } - if (pEnv && *pEnv) { - return TSDB_CODE_SUCCESS; - } - - if (tsdbLockRepo(pTsdb) != 0) { - return TSDB_CODE_FAILED; - } - if (*pEnv == NULL) { - if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) { - tsdbUnlockRepo(pTsdb); + if ((*pEnv = tsdbNewSmaEnv(pTsdb, path, did)) == NULL) { return TSDB_CODE_FAILED; } } - if (tsdbUnlockRepo(pTsdb) != 0) { - tsdbFreeSmaEnv(*pEnv); - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; } @@ -170,7 +181,7 @@ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { tsdbDestroySmaState(pSmaEnv->pStat); tfree(pSmaEnv->pStat); tfree(pSmaEnv->path); - pthread_rwlock_destroy(&(pSmaEnv->lock)); + taosThreadRwlockDestroy(&(pSmaEnv->lock)); tsdbCloseBDBEnv(pSmaEnv->dbEnv); } } @@ -181,6 +192,21 @@ void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) { return NULL; } +static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { + if (pStat == NULL) return 0; + int ref = T_REF_INC(pStat); + tsdbDebug("vgId:%d ref sma stat %p ref %d", REPO_ID(pTsdb), pStat, ref); + return 0; +} + +static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { + if (pStat == NULL) return 0; + + int ref = T_REF_DEC(pStat); + tsdbDebug("vgId:%d unref sma stat %p ref %d", REPO_ID(pTsdb), pStat, ref); + return 0; +} + static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { ASSERT(pSmaStat != NULL); @@ -188,11 +214,14 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { return TSDB_CODE_SUCCESS; } - // TODO: lock. lazy mode when update expired window, or hungry mode during tsdbNew. + /** + * 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tsdbNew). + * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if + * tsdbInitSmaStat invoked in other multithread environment later. + */ if (*pSmaStat == NULL) { *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat)); if (*pSmaStat == NULL) { - // TODO: unlock terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } @@ -202,11 +231,9 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { if ((*pSmaStat)->smaStatItems == NULL) { tfree(*pSmaStat); - // TODO: unlock return TSDB_CODE_FAILED; } } - // TODO: unlock return TSDB_CODE_SUCCESS; } @@ -234,16 +261,73 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { if (pSmaStat) { // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL); + void *item = taosHashIterate(pSmaStat->smaStatItems, NULL); while (item != NULL) { - tfree(item->pSma); - taosHashCleanup(item->expiredWindows); + SSmaStatItem *pItem = *(SSmaStatItem **)item; + if (pItem != NULL) { + tdDestroyTSma(pItem->pSma); + tfree(pItem->pSma); + taosHashCleanup(pItem->expiredWindows); + tfree(pItem); + } item = taosHashIterate(pSmaStat->smaStatItems, item); } taosHashCleanup(pSmaStat->smaStatItems); } } +static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { + SSmaEnv *pEnv = NULL; + + // return if already init + switch (smaType) { + case TSDB_SMA_TYPE_TIME_RANGE: + if ((pEnv = (SSmaEnv *)atomic_load_ptr(&pTsdb->pTSmaEnv)) != NULL) { + return TSDB_CODE_SUCCESS; + } + break; + case TSDB_SMA_TYPE_ROLLUP: + if ((pEnv = (SSmaEnv *)atomic_load_ptr(&pTsdb->pRSmaEnv)) != NULL) { + return TSDB_CODE_SUCCESS; + } + break; + default: + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + + // init sma env + tsdbLockRepo(pTsdb); + pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&pTsdb->pTSmaEnv) : atomic_load_ptr(&pTsdb->pRSmaEnv); + if (pEnv == NULL) { + char rname[TSDB_FILENAME_LEN] = {0}; + + SDiskID did = {0}; + tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did); + if (did.level < 0 || did.id < 0) { + tsdbUnlockRepo(pTsdb); + return TSDB_CODE_FAILED; + } + tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname); + + if (tfsMkdirRecurAt(pTsdb->pTfs, rname, did) != TSDB_CODE_SUCCESS) { + tsdbUnlockRepo(pTsdb); + return TSDB_CODE_FAILED; + } + + if (tsdbInitSmaEnv(pTsdb, rname, did, &pEnv) != TSDB_CODE_SUCCESS) { + tsdbUnlockRepo(pTsdb); + return TSDB_CODE_FAILED; + } + + (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv) + : atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv); + } + tsdbUnlockRepo(pTsdb); + + return TSDB_CODE_SUCCESS; +}; + /** * @brief Update expired window according to msg from stream computing module. * @@ -252,47 +336,41 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { * @param msg * @return int32_t */ -int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - SSmaEnv * pEnv = NULL; - +int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) { if (!msg || !pTsdb->pMeta) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } - char smaPath[TSDB_FILENAME_LEN] = "/proj/.sma/"; - if (tsdbInitSmaEnv(pTsdb, smaPath, &pEnv) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } - - if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - pTsdb->pTSmaEnv = pEnv; - } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - pTsdb->pRSmaEnv = pEnv; - } else { - ASSERT(0); - } - - // TODO: decode the msg => start - int64_t indexUid = SMA_TEST_INDEX_UID; - // const char * indexName = SMA_TEST_INDEX_NAME; + // TODO: decode the msg from Stream Computing module => start + int64_t indexUid = SMA_TEST_INDEX_UID; const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; - int64_t now = taosGetTimestampMs(); + TSKEY skey1 = 1646987196 * 1e3; for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { - expiredWindows[i] = now + i; + expiredWindows[i] = skey1 + i; } - // TODO: decode the msg <= end + + if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + SSmaEnv * pEnv = REPO_SMA_ENV(pTsdb, smaType); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL); + + tsdbRefSmaStat(pTsdb, pStat); + SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); if (pItem == NULL) { pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state if (pItem == NULL) { // Response to stream computing: OOM - // For query, if the indexName not found, the TSDB should tell query module to query raw TS data. + // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } @@ -302,29 +380,25 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); free(pItem); + tsdbUnRefSmaStat(pTsdb, pStat); tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, tstrerror(terrno)); return TSDB_CODE_FAILED; } pItem->pSma = pSma; - // TODO: change indexName to indexUid if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { // If error occurs during put smaStatItem, free the resources of pItem taosHashCleanup(pItem->expiredWindows); free(pItem); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } } -#if 0 - SSmaStatItem *pItem1 = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - int size1 = taosHashGetSize(pItem1->expiredWindows); - tsdbWarn("vgId:%d smaIndex %" PRIi64 " size is %d before hashPut", REPO_ID(pTsdb), indexUid, size1); -#endif int8_t state = TSDB_SMA_STAT_EXPIRED; for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { - if (taosHashPut(pItem->expiredWindows, &expiredWindows[i], sizeof(TSKEY), &state, sizeof(state)) != 0) { + if (taosHashPut(pItem->expiredWindows, expiredWindows + i, sizeof(TSKEY), &state, sizeof(state)) != 0) { // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would // tell query module to query raw TS data. // N.B. @@ -334,37 +408,43 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) { taosHashCleanup(pItem->expiredWindows); tfree(pItem->pSma); taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } + tsdbDebug("vgId:%d smaIndex %" PRIi64 " tsKey %" PRIi64 " is put to hash", REPO_ID(pTsdb), indexUid, + expiredWindows[i]); } -#if 0 - SSmaStatItem *pItem2 = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - int size2 = taosHashGetSize(pItem1->expiredWindows); - tsdbWarn("vgId:%d smaIndex %" PRIi64 " size is %d after hashPut", REPO_ID(pTsdb), indexUid, size2); -#endif - + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; } -static int32_t tsdbResetExpiredWindow(SSmaStat *pStat, int64_t indexUid, TSKEY skey) { +static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { SSmaStatItem *pItem = NULL; - // TODO: If HASH_ENTRY_LOCK used, whether rwlock needed to handle cases of removing hashNode? + tsdbRefSmaStat(pTsdb, pStat); + if (pStat && pStat->smaStatItems) { - pItem = (SSmaStatItem *)taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); } -#if 0 - if (pItem != NULL) { - // TODO: reset time window for the sma data blocks + if ((pItem != NULL) && ((pItem = *(SSmaStatItem **)pItem) != NULL)) { + // pItem resides in hash buffer all the time unless drop sma index + // TODO: multithread protect if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { // error handling + tsdbUnRefSmaStat(pTsdb, pStat); + tsdbWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " failed", REPO_ID(pTsdb), + skey, indexUid); + return TSDB_CODE_FAILED; } - } else { // error handling + tsdbUnRefSmaStat(pTsdb, pStat); + tsdbWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, REPO_ID(pTsdb), skey, indexUid); + return TSDB_CODE_FAILED; } -#endif + + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; } @@ -378,32 +458,32 @@ static int32_t tsdbResetExpiredWindow(SSmaStat *pStat, int64_t indexUid, TSKEY s static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { // TODO: configurable for SMA_STORAGE_SPLIT_HOURS? switch (intervalUnit) { - case TD_TIME_UNIT_HOUR: + case TIME_UNIT_HOUR: if (interval < SMA_STORAGE_SPLIT_HOURS) { return SMA_STORAGE_LEVEL_DFILESET; } break; - case TD_TIME_UNIT_MINUTE: + case TIME_UNIT_MINUTE: if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) { return SMA_STORAGE_LEVEL_DFILESET; } break; - case TD_TIME_UNIT_SEC: + case TIME_UNIT_SECOND: if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) { return SMA_STORAGE_LEVEL_DFILESET; } break; - case TD_TIME_UNIT_MILLISEC: + case TIME_UNIT_MILLISECOND: if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) { return SMA_STORAGE_LEVEL_DFILESET; } break; - case TD_TIME_UNIT_MICROSEC: + case TIME_UNIT_MICROSECOND: if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) { return SMA_STORAGE_LEVEL_DFILESET; } break; - case TD_TIME_UNIT_NANOSEC: + case TIME_UNIT_NANOSECOND: if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) { return SMA_STORAGE_LEVEL_DFILESET; } @@ -418,7 +498,7 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { * @brief Insert TSma data blocks to DB File build by B+Tree * * @param pSmaH - * @param smaKey + * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) * @param keyLen * @param pData * @param dataLen @@ -426,89 +506,95 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { */ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { SDBFile *pDBFile = &pSmaH->dFile; - - // TODO: insert sma data blocks into B+Tree tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d", - REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), - *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); + REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), + *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); + // TODO: insert sma data blocks into B+Tree(TDB) if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) { return TSDB_CODE_FAILED; } -#ifdef SMA_PRINT_DEBUG_LOG +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ uint32_t valueSize = 0; void * data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); ASSERT(data != NULL); for (uint32_t v = 0; v < valueSize; v += 8) { - tsdbWarn("vgId:%d sma data - val[%d] is %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); + tsdbWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); } #endif return TSDB_CODE_SUCCESS; } +/** + * @brief Approximate value for week/month/year. + * + * @param interval + * @param intervalUnit + * @param precision + * @return int64_t + */ static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision) { - if (intervalUnit < TD_TIME_UNIT_MILLISEC) { - switch (intervalUnit) { - case TD_TIME_UNIT_YEAR: - case TD_TIME_UNIT_SEASON: - case TD_TIME_UNIT_MONTH: - case TD_TIME_UNIT_WEEK: - // illegal time unit - tsdbError("invalid interval unit: %d\n", intervalUnit); - TASSERT(0); - break; - case TD_TIME_UNIT_DAY: // the interval for tSma calculation must <= day - interval *= 86400 * 1e3; - break; - case TD_TIME_UNIT_HOUR: - interval *= 3600 * 1e3; - break; - case TD_TIME_UNIT_MINUTE: - interval *= 60 * 1e3; - break; - case TD_TIME_UNIT_SEC: - interval *= 1e3; - break; - default: - break; - } + switch (intervalUnit) { + case TIME_UNIT_YEAR: // approximate value + interval *= 365 * 86400 * 1e3; + break; + case TIME_UNIT_MONTH: // approximate value + interval *= 30 * 86400 * 1e3; + break; + case TIME_UNIT_WEEK: // approximate value + interval *= 7 * 86400 * 1e3; + break; + case TIME_UNIT_DAY: // the interval for tSma calculation must <= day + interval *= 86400 * 1e3; + break; + case TIME_UNIT_HOUR: + interval *= 3600 * 1e3; + break; + case TIME_UNIT_MINUTE: + interval *= 60 * 1e3; + break; + case TIME_UNIT_SECOND: + interval *= 1e3; + break; + default: + break; } switch (precision) { case TSDB_TIME_PRECISION_MILLI: - if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us return interval / 1e3; - } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second return interval / 1e6; - } else { + } else { // ms return interval; } break; case TSDB_TIME_PRECISION_MICRO: - if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us return interval; - } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns return interval / 1e3; - } else { + } else { // ms return interval * 1e3; } break; case TSDB_TIME_PRECISION_NANO: - if (TD_TIME_UNIT_MICROSEC == intervalUnit) { + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us return interval * 1e3; - } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns return interval; - } else { + } else { // ms return interval * 1e6; } break; default: // ms - if (TD_TIME_UNIT_MICROSEC == intervalUnit) { // us + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us return interval / 1e3; - } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) { // nano second + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns return interval / 1e6; - } else { + } else { // ms return interval; } break; @@ -577,12 +663,15 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { } } -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) { +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL); + + pSmaH->dFile.fid = fid; char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid); + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); pSmaH->dFile.path = strdup(tSmaFile); + return TSDB_CODE_SUCCESS; } @@ -621,8 +710,9 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { STsdbCfg * pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; + SSmaEnv * pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); - if (!pTsdb->pTSmaEnv) { + if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return terrno; @@ -640,6 +730,17 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return TSDB_CODE_FAILED; } + int64_t indexUid = SMA_TEST_INDEX_UID; + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + } + // Step 1: Judge the storage level and days int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); @@ -648,7 +749,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file // - Set and open the DFile or the B+Tree file // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); + tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); @@ -664,7 +765,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // TODO:tsdbEndTSmaCommit(); // Step 3: reset the SSmaStat - tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); + tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); tsdbDestroyTSmaWriteH(&tSmaH); return TSDB_CODE_SUCCESS; @@ -710,7 +811,7 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { // TODO:tsdbEndTSmaCommit(); // reset the SSmaStat - tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey); + tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey); return TSDB_CODE_SUCCESS; } @@ -735,13 +836,16 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interv * @brief Init of tSma FS * * @param pReadH + * @param indexUid * @param skey * @return int32_t */ -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pSmaH->pTsdb)->precision)); +static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { + STsdb *pTsdb = pSmaH->pTsdb; + + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pTsdb)->precision)); char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pSmaH->pTsdb), fid); + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); pSmaH->dFile.path = strdup(tSmaFile); pSmaH->smaFsIter.iter = 0; pSmaH->smaFsIter.fid = fid; @@ -798,12 +902,25 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { * @return int32_t */ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, - int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, + int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) { - SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid)); - if (pItem == NULL) { - // mark all window as expired and notify query module to query raw TS data. - return TSDB_CODE_SUCCESS; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + return TSDB_CODE_FAILED; + } + + tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pEnv)); + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) { + // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if + // it's NULL. + tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, REPO_ID(pTsdb), indexUid); + return TSDB_CODE_FAILED; } #if 0 @@ -815,16 +932,25 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ } } #endif -#if 0 - if (taosHashGet(pItem->expiredWindows, &querySkey, sizeof(TSKEY)) != NULL) { + +#if 1 + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY)) != NULL) { // TODO: mark this window as expired. + tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb), + querySKey, indexUid); + } else { + tsdbDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, REPO_ID(pTsdb), querySKey, + indexUid); } + tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); + #endif + STSmaReadH tReadH = {0}; tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit); tsdbCloseDBF(&tReadH.dFile); - tsdbInitTSmaFile(&tReadH, querySkey); + tsdbInitTSmaFile(&tReadH, indexUid, querySKey); if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) { tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); return TSDB_CODE_FAILED; @@ -832,11 +958,11 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ char smaKey[SMA_KEY_LEN] = {0}; void *pSmaKey = &smaKey; - tsdbEncodeTSmaKey(tableUid, colId, querySkey, (void **)&pSmaKey); + tsdbEncodeTSmaKey(tableUid, colId, querySKey, (void **)&pSmaKey); tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), - tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), - *(int64_t *)POINTER_SHIFT(smaKey, 10), SMA_KEY_LEN); + tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), + *(int64_t *)POINTER_SHIFT(smaKey, 10), SMA_KEY_LEN); void * result = NULL; uint32_t valueSize = 0; @@ -848,12 +974,14 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tsdbCloseDBF(&tReadH.dFile); return TSDB_CODE_FAILED; } - tfree(result); -#ifdef SMA_PRINT_DEBUG_LOG + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ for (uint32_t v = 0; v < valueSize; v += 8) { - tsdbWarn("vgId:%d v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); + tsdbWarn("vgId:%d get sma data v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); } #endif + tfree(result); // TODO: fill the result to output + #if 0 int32_t nResult = 0; int64_t lastKey = 0; @@ -947,7 +1075,6 @@ int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) { * @brief Insert Time-range-wise Rollup Sma(RSma) data * * @param pTsdb - * @param param * @param msg * @return int32_t */ @@ -960,9 +1087,9 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { } int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, - tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, int32_t nMaxResult) { + tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) { int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, interval, intervalUnit, tableUid, colId, querySkey, + if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, interval, intervalUnit, tableUid, colId, querySKey, nMaxResult)) < 0) { tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 9d1877bdb7dc3b5b4b0b71ea92e85d2056fc60c5..3b43fd82f6973f252a833fb03eb54ff201bb55bd 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -19,8 +19,8 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - pthread_mutex_t mutex; - pthread_cond_t hasFree; + TdThreadMutex mutex; + TdThreadCond hasFree; TD_DLIST(SVMemAllocator) free; TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 2a3862c7cbf37de74506bdf217fe62ccdcfd6e52..c7405fdceaf1caa0ea04015249b044b06de561b1 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -27,7 +27,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnodeCfg cfg = defaultVnodeOptions; if (pVnodeCfg != NULL) { cfg.vgId = pVnodeCfg->vgId; - cfg.pDnode = pVnodeCfg->pDnode; + cfg.pWrapper = pVnodeCfg->pWrapper; cfg.pTfs = pVnodeCfg->pTfs; cfg.dbId = pVnodeCfg->dbId; cfg.hashBegin = pVnodeCfg->hashBegin; @@ -79,7 +79,7 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { } pVnode->vgId = pVnodeCfg->vgId; - pVnode->pDnode = pVnodeCfg->pDnode; + pVnode->pWrapper = pVnodeCfg->pWrapper; pVnode->pTfs = pVnodeCfg->pTfs; pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 477deed8c8944fc511c8aa3f78951d2131171f40..442921b90ea30dfef491c96e5c4e6b924c30d89d 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -25,23 +25,23 @@ int vnodeInit(const SVnodeOpt *pOption) { } vnodeMgr.stop = false; - vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; - vnodeMgr.sendReqToDnodeFp = pOption->sendReqToDnodeFp; + vnodeMgr.putToQueryQFp = pOption->putToQueryQFp; + vnodeMgr.sendReqFp = pOption->sendReqFp; // Start commit handers if (pOption->nthreads > 0) { vnodeMgr.nthreads = pOption->nthreads; - vnodeMgr.threads = (pthread_t*)calloc(pOption->nthreads, sizeof(pthread_t)); + vnodeMgr.threads = (TdThread*)calloc(pOption->nthreads, sizeof(TdThread)); if (vnodeMgr.threads == NULL) { return -1; } - pthread_mutex_init(&(vnodeMgr.mutex), NULL); - pthread_cond_init(&(vnodeMgr.hasTask), NULL); + taosThreadMutexInit(&(vnodeMgr.mutex), NULL); + taosThreadCondInit(&(vnodeMgr.hasTask), NULL); TD_DLIST_INIT(&(vnodeMgr.queue)); for (uint16_t i = 0; i < pOption->nthreads; i++) { - pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); + taosThreadCreate(&(vnodeMgr.threads[i]), NULL, loop, NULL); // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } } else { @@ -63,42 +63,42 @@ void vnodeCleanup() { } // Stop commit handler - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); vnodeMgr.stop = true; - pthread_cond_broadcast(&(vnodeMgr.hasTask)); - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadCondBroadcast(&(vnodeMgr.hasTask)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); for (uint16_t i = 0; i < vnodeMgr.nthreads; i++) { - pthread_join(vnodeMgr.threads[i], NULL); + taosThreadJoin(vnodeMgr.threads[i], NULL); } tfree(vnodeMgr.threads); - pthread_cond_destroy(&(vnodeMgr.hasTask)); - pthread_mutex_destroy(&(vnodeMgr.mutex)); + taosThreadCondDestroy(&(vnodeMgr.hasTask)); + taosThreadMutexDestroy(&(vnodeMgr.mutex)); } int vnodeScheduleTask(SVnodeTask* pTask) { - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); TD_DLIST_APPEND(&(vnodeMgr.queue), pTask); - pthread_cond_signal(&(vnodeMgr.hasTask)); + taosThreadCondSignal(&(vnodeMgr.hasTask)); - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); return 0; } -int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { - if (pVnode == NULL || pVnode->pDnode == NULL || vnodeMgr.putReqToVQueryQFp == NULL) { +int32_t vnodePutToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { + if (pVnode == NULL || pVnode->pMeta == NULL || vnodeMgr.putToQueryQFp == NULL) { terrno = TSDB_CODE_VND_APP_ERROR; return -1; } - return (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); + return (*vnodeMgr.putToQueryQFp)(pVnode->pWrapper, pReq); } -void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { - (*vnodeMgr.sendReqToDnodeFp)(pVnode->pDnode, epSet, pReq); +void vnodeSendReq(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { + (*vnodeMgr.sendReqFp)(pVnode->pWrapper, epSet, pReq); } /* ------------------------ STATIC METHODS ------------------------ */ @@ -107,15 +107,15 @@ static void* loop(void* arg) { SVnodeTask* pTask; for (;;) { - pthread_mutex_lock(&(vnodeMgr.mutex)); + taosThreadMutexLock(&(vnodeMgr.mutex)); for (;;) { pTask = TD_DLIST_HEAD(&(vnodeMgr.queue)); if (pTask == NULL) { if (vnodeMgr.stop) { - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); return NULL; } else { - pthread_cond_wait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); + taosThreadCondWait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); } } else { TD_DLIST_POP(&(vnodeMgr.queue), pTask); @@ -123,7 +123,7 @@ static void* loop(void* arg) { } } - pthread_mutex_unlock(&(vnodeMgr.mutex)); + taosThreadMutexUnlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); free(pTask); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 67f212c2736579f4c95c573ac344ba1ed31cc038..0ac60ea72d4689f392e23bf1506206a54ec93554 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -21,7 +21,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, - (putReqToQueryQFp)vnodePutReqToVQueryQ, (sendReqToDnodeFp)vnodeSendReqToDnode); + (putReqToQueryQFp)vnodePutToVQueryQ, (sendReqFp)vnodeSendReq); } void vnodeQueryClose(SVnode *pVnode) { diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index f82105aba0531bb53794376659b4168fcd57fa86..64747dea8892d27994037a4097488935c5b06fcb 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -17,17 +17,19 @@ #include "vnd.h" int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { - SRpcMsg *pMsg; + SNodeMsg *pMsg; + SRpcMsg *pRpc; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); + pMsg = *(SNodeMsg **)taosArrayGet(pMsgs, i); + pRpc = &pMsg->rpcMsg; // set request version - void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pRpc->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedI64(&pBuf, ver); - if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { + if (walWrite(pVnode->pWal, ver, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) { // TODO: handle error /*ASSERT(false);*/ vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr()); @@ -41,7 +43,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { return 0; } -int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { void *ptr = NULL; if (pVnode->config.streamMode == 0) { @@ -63,7 +65,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { switch (pMsg->msgType) { case TDMT_VND_CREATE_STB: { - SVCreateTbReq vCreateTbReq = {0}; + SVCreateTbReq vCreateTbReq = {0}; tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { // TODO: handle error @@ -128,7 +130,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { break; } case TDMT_VND_ALTER_STB: { - SVCreateTbReq vAlterTbReq = {0}; + SVCreateTbReq vAlterTbReq = {0}; vTrace("vgId:%d, process alter stb req", pVnode->vgId); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); free(vAlterTbReq.stbCfg.pSchema); @@ -160,6 +162,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { } } break; + case TDMT_VND_TASK_DEPLOY: { + if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { + } + } break; case TDMT_VND_CREATE_SMA: { // timeRangeSMA SSmaCfg vCreateSmaReq = {0}; if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) { @@ -167,6 +174,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return -1; } + // record current timezone of server side + tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezone, TD_TIMEZONE_LEN); + if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) { // TODO: handle error tdDestroyTSma(&vCreateSmaReq.tSma); diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 18dca33bdaf73e4ebcd95a1623960514b26667a3..8140113e673cda3f89673344751b546821a88a5d 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -14,13 +14,13 @@ */ #include +#include #include #include #include #include #include -#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -33,13 +33,14 @@ int main(int argc, char **argv) { return RUN_ALL_TESTS(); } +#if 1 TEST(testCase, tSma_Meta_Encode_Decode_Test) { // encode STSma tSma = {0}; tSma.version = 0; - tSma.intervalUnit = TD_TIME_UNIT_DAY; + tSma.intervalUnit = TIME_UNIT_DAY; tSma.interval = 1; - tSma.slidingUnit = TD_TIME_UNIT_HOUR; + tSma.slidingUnit = TIME_UNIT_HOUR; tSma.sliding = 0; tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN); tstrncpy(tSma.timezone, "Asia/Shanghai", TD_TIMEZONE_LEN); @@ -49,44 +50,46 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma}; uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper); - void *buf = calloc(bufLen, 1); - assert(buf != NULL); + void *buf = calloc(1, bufLen); + ASSERT_NE(buf, nullptr); STSmaWrapper *pSW = (STSmaWrapper *)buf; uint32_t len = tEncodeTSmaWrapper(&buf, &tSmaWrapper); - EXPECT_EQ(len, bufLen); + ASSERT_EQ(len, bufLen); // decode STSmaWrapper dstTSmaWrapper = {0}; void * result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper); - assert(result != NULL); + ASSERT_NE(result, nullptr); - EXPECT_EQ(tSmaWrapper.number, dstTSmaWrapper.number); + ASSERT_EQ(tSmaWrapper.number, dstTSmaWrapper.number); for (int i = 0; i < tSmaWrapper.number; ++i) { STSma *pSma = tSmaWrapper.tSma + i; STSma *qSma = dstTSmaWrapper.tSma + i; - EXPECT_EQ(pSma->version, qSma->version); - EXPECT_EQ(pSma->intervalUnit, qSma->intervalUnit); - EXPECT_EQ(pSma->slidingUnit, qSma->slidingUnit); - EXPECT_STRCASEEQ(pSma->indexName, qSma->indexName); - EXPECT_STRCASEEQ(pSma->timezone, qSma->timezone); - EXPECT_EQ(pSma->indexUid, qSma->indexUid); - EXPECT_EQ(pSma->tableUid, qSma->tableUid); - EXPECT_EQ(pSma->interval, qSma->interval); - EXPECT_EQ(pSma->sliding, qSma->sliding); - EXPECT_EQ(pSma->exprLen, qSma->exprLen); - EXPECT_STRCASEEQ(pSma->expr, qSma->expr); - EXPECT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen); - EXPECT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter); + ASSERT_EQ(pSma->version, qSma->version); + ASSERT_EQ(pSma->intervalUnit, qSma->intervalUnit); + ASSERT_EQ(pSma->slidingUnit, qSma->slidingUnit); + ASSERT_STRCASEEQ(pSma->indexName, qSma->indexName); + ASSERT_STRCASEEQ(pSma->timezone, qSma->timezone); + ASSERT_EQ(pSma->indexUid, qSma->indexUid); + ASSERT_EQ(pSma->tableUid, qSma->tableUid); + ASSERT_EQ(pSma->interval, qSma->interval); + ASSERT_EQ(pSma->sliding, qSma->sliding); + ASSERT_EQ(pSma->exprLen, qSma->exprLen); + ASSERT_STRCASEEQ(pSma->expr, qSma->expr); + ASSERT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen); + ASSERT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter); } // resource release + tfree(pSW); tdDestroyTSma(&tSma); tdDestroyTSmaWrapper(&dstTSmaWrapper); } +#endif #if 1 TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { @@ -103,9 +106,9 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { // encode STSma tSma = {0}; tSma.version = 0; - tSma.intervalUnit = TD_TIME_UNIT_DAY; + tSma.intervalUnit = TIME_UNIT_DAY; tSma.interval = 1; - tSma.slidingUnit = TD_TIME_UNIT_HOUR; + tSma.slidingUnit = TIME_UNIT_HOUR; tSma.sliding = 0; tSma.indexUid = indexUid1; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); @@ -113,11 +116,13 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { tSma.tableUid = tbUid; tSma.exprLen = strlen(expr); - tSma.expr = (char *)calloc(tSma.exprLen + 1, 1); + tSma.expr = (char *)calloc(1, tSma.exprLen + 1); + ASSERT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); + ASSERT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); SMeta * pMeta = NULL; @@ -129,18 +134,18 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { pMeta = metaOpen(smaTestDir, pMetaCfg, NULL); assert(pMeta != NULL); // save index 1 - EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); + ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); pSmaCfg->indexUid = indexUid2; tstrncpy(pSmaCfg->indexName, smaIndexName2, TSDB_INDEX_NAME_LEN); pSmaCfg->version = 1; - pSmaCfg->intervalUnit = TD_TIME_UNIT_HOUR; + pSmaCfg->intervalUnit = TIME_UNIT_HOUR; pSmaCfg->interval = 1; - pSmaCfg->slidingUnit = TD_TIME_UNIT_MINUTE; + pSmaCfg->slidingUnit = TIME_UNIT_MINUTE; pSmaCfg->sliding = 5; // save index 2 - EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); + ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); // get value by indexName STSma *qSmaCfg = NULL; @@ -150,8 +155,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("timezone1 = %s\n", qSmaCfg->timezone); printf("expr1 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); - EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); - EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); + ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); + ASSERT_EQ(qSmaCfg->tableUid, tSma.tableUid); tdDestroyTSma(qSmaCfg); tfree(qSmaCfg); @@ -161,8 +166,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("timezone2 = %s\n", qSmaCfg->timezone); printf("expr2 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); - EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); - EXPECT_EQ(qSmaCfg->interval, tSma.interval); + ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); + ASSERT_EQ(qSmaCfg->interval, tSma.interval); tdDestroyTSma(qSmaCfg); tfree(qSmaCfg); @@ -178,25 +183,25 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("indexName = %s\n", indexName); ++indexCnt; } - EXPECT_EQ(indexCnt, nCntTSma); + ASSERT_EQ(indexCnt, nCntTSma); metaCloseSmaCurosr(pSmaCur); // get wrapper by table uid STSmaWrapper *pSW = metaGetSmaInfoByTable(pMeta, tbUid); assert(pSW != NULL); - EXPECT_EQ(pSW->number, nCntTSma); - EXPECT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); - EXPECT_STRCASEEQ(pSW->tSma->timezone, timezone); - EXPECT_STRCASEEQ(pSW->tSma->expr, expr); - EXPECT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter); - EXPECT_EQ(pSW->tSma->indexUid, indexUid1); - EXPECT_EQ(pSW->tSma->tableUid, tbUid); - EXPECT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); - EXPECT_STRCASEEQ((pSW->tSma + 1)->timezone, timezone); - EXPECT_STRCASEEQ((pSW->tSma + 1)->expr, expr); - EXPECT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter); - EXPECT_EQ((pSW->tSma + 1)->indexUid, indexUid2); - EXPECT_EQ((pSW->tSma + 1)->tableUid, tbUid); + ASSERT_EQ(pSW->number, nCntTSma); + ASSERT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); + ASSERT_STRCASEEQ(pSW->tSma->timezone, timezone); + ASSERT_STRCASEEQ(pSW->tSma->expr, expr); + ASSERT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter); + ASSERT_EQ(pSW->tSma->indexUid, indexUid1); + ASSERT_EQ(pSW->tSma->tableUid, tbUid); + ASSERT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); + ASSERT_STRCASEEQ((pSW->tSma + 1)->timezone, timezone); + ASSERT_STRCASEEQ((pSW->tSma + 1)->expr, expr); + ASSERT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter); + ASSERT_EQ((pSW->tSma + 1)->indexUid, indexUid2); + ASSERT_EQ((pSW->tSma + 1)->tableUid, tbUid); tdDestroyTSmaWrapper(pSW); tfree(pSW); @@ -208,7 +213,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("metaGetSmaTbUids: uid[%" PRIu32 "] = %" PRIi64 "\n", i, *(tb_uid_t *)taosArrayGet(pUids, i)); // printf("metaGetSmaTbUids: index[%" PRIu32 "] = %s", i, (char *)taosArrayGet(pUids, i)); } - EXPECT_EQ(taosArrayGetSize(pUids), 1); + ASSERT_EQ(taosArrayGetSize(pUids), 1); taosArrayDestroy(pUids); // resource release @@ -231,7 +236,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { const tb_uid_t tbUid = 1234567890; const int64_t indexUid1 = 2000000001; const int64_t interval1 = 1; - const int8_t intervalUnit1 = TD_TIME_UNIT_DAY; + const int8_t intervalUnit1 = TIME_UNIT_DAY; const uint32_t nCntTSma = 2; TSKEY skey1 = 1646987196; const int64_t testSmaData1 = 100; @@ -239,9 +244,9 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { // encode STSma tSma = {0}; tSma.version = 0; - tSma.intervalUnit = TD_TIME_UNIT_DAY; + tSma.intervalUnit = TIME_UNIT_DAY; tSma.interval = 1; - tSma.slidingUnit = TD_TIME_UNIT_HOUR; + tSma.slidingUnit = TIME_UNIT_HOUR; tSma.sliding = 0; tSma.indexUid = indexUid1; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); @@ -249,11 +254,13 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { tSma.tableUid = tbUid; tSma.exprLen = strlen(expr); - tSma.expr = (char *)calloc(tSma.exprLen + 1, 1); + tSma.expr = (char *)calloc(1, tSma.exprLen + 1); + ASSERT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); - tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); + tSma.tagsFilter = (char *)calloc(1, tSma.tagsFilterLen + 1); + ASSERT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); SMeta * pMeta = NULL; @@ -265,24 +272,24 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { pMeta = metaOpen(smaTestDir, pMetaCfg, NULL); assert(pMeta != NULL); // save index 1 - EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); + ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); // step 2: insert data STSmaDataWrapper *pSmaData = NULL; - STsdb tsdb = {0}; - STsdbCfg * pCfg = &tsdb.config; - - tsdb.pMeta = pMeta; - tsdb.vgId = 2; - tsdb.config.daysPerFile = 10; // default days is 10 - tsdb.config.keep1 = 30; - tsdb.config.keep2 = 90; - tsdb.config.keep = 365; - tsdb.config.precision = TSDB_TIME_PRECISION_MILLI; - tsdb.config.update = TD_ROW_OVERWRITE_UPDATE; - tsdb.config.compression = TWO_STAGE_COMP; - - switch (tsdb.config.precision) { + STsdb * pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); + STsdbCfg * pCfg = &pTsdb->config; + + pTsdb->pMeta = pMeta; + pTsdb->vgId = 2; + pTsdb->config.daysPerFile = 10; // default days is 10 + pTsdb->config.keep1 = 30; + pTsdb->config.keep2 = 90; + pTsdb->config.keep = 365; + pTsdb->config.precision = TSDB_TIME_PRECISION_MILLI; + pTsdb->config.update = TD_ROW_OVERWRITE_UPDATE; + pTsdb->config.compression = TWO_STAGE_COMP; + + switch (pTsdb->config.precision) { case TSDB_TIME_PRECISION_MILLI: skey1 *= 1e3; break; @@ -297,19 +304,26 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { break; } - char *msg = (char *)calloc(100, 1); - EXPECT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); + SDiskCfg pDisks = {.level = 0, .primary = 1}; + strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN); + int32_t numOfDisks = 1; + pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); + ASSERT_NE(pTsdb->pTfs, nullptr); + + char *msg = (char *)calloc(1, 100); + ASSERT_NE(msg, nullptr); + ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); // init int32_t allocCnt = 0; - int32_t allocStep = 40960; - int32_t buffer = 4096; + int32_t allocStep = 16384; + int32_t buffer = 1024; void * buf = NULL; - EXPECT_EQ(tsdbMakeRoom(&buf, allocStep), 0); + ASSERT_EQ(tsdbMakeRoom(&buf, allocStep), 0); int32_t bufSize = taosTSizeof(buf); int32_t numOfTables = 10; col_id_t numOfCols = 4096; - EXPECT_GT(numOfCols, 0); + ASSERT_GT(numOfCols, 0); pSmaData = (STSmaDataWrapper *)buf; printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData); @@ -326,7 +340,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { int32_t tableDataLen = sizeof(STSmaTbData); for (col_id_t c = 0; c < numOfCols; ++c) { if (bufSize - len - tableDataLen < buffer) { - EXPECT_EQ(tsdbMakeRoom(&buf, bufSize + allocStep), 0); + ASSERT_EQ(tsdbMakeRoom(&buf, bufSize + allocStep), 0); pSmaData = (STSmaDataWrapper *)buf; pTbData = (STSmaTbData *)POINTER_SHIFT(pSmaData, len); bufSize = taosTSizeof(buf); @@ -353,28 +367,31 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { } pSmaData->dataLen = (len - sizeof(STSmaDataWrapper)); - EXPECT_GE(bufSize, pSmaData->dataLen); + ASSERT_GE(bufSize, pSmaData->dataLen); // execute - EXPECT_EQ(tsdbInsertTSmaData(&tsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); + ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); // step 3: query uint32_t checkDataCnt = 0; for (int32_t t = 0; t < numOfTables; ++t) { for (col_id_t c = 0; c < numOfCols; ++c) { - EXPECT_EQ(tsdbGetTSmaData(&tsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t, + ASSERT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t, c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1), TSDB_CODE_SUCCESS); ++checkDataCnt; } } - + printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt); // release data + tfree(msg); taosTZfree(buf); // release meta tdDestroyTSma(&tSma); + tfsClose(pTsdb->pTfs); + tsdbClose(pTsdb); metaClose(pMeta); } #endif diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index d766f3cbf12b949dd82b83be68f757b1e2a67ab8..ea8195bfd11081dd506ee61302a8141c99038b02 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(transport) add_subdirectory(sync) -add_subdirectory(tdb) +# add_subdirectory(tdb) add_subdirectory(index) add_subdirectory(wal) add_subdirectory(parser) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 837bac1f1b261a716d028bda9e0cfc323a301db0..ca7d4dbe2cb4881d5c34f47413d3646113c3cf38 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -186,7 +186,7 @@ typedef struct SCatalogMgmt { bool exit; SRWLatch lock; SCtgQueue queue; - pthread_t updateThread; + TdThread updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 16b4931eb5c7bb2a6ccddd1ddc55daece7f28229..37f3fe77a3ad5ee8bbefd181aec066f329c4a985 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1174,7 +1174,7 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { - int16_t widx = abs(id % mgmt->slotNum); + int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; @@ -1204,11 +1204,11 @@ _return: } int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { - int16_t widx = abs(id % mgmt->slotNum); + int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; - + CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); @@ -1245,7 +1245,7 @@ _return: } int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { - int16_t widx = abs(id % mgmt->slotNum); + int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; @@ -2174,16 +2174,16 @@ void* ctgUpdateThreadFunc(void* param) { int32_t ctgStartUpdateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); CTG_ERR_RET(terrno); } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); return TSDB_CODE_SUCCESS; } @@ -2530,6 +2530,10 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet) { + +} + int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_API_ENTER(); @@ -2593,6 +2597,9 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, const char *pIndexName, SIndexMeta** pIndexMeta) { + +} int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); @@ -2800,12 +2807,15 @@ _return: int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { CTG_API_ENTER(); - + + int32_t code = 0; if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - //TODO + CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, &pQnodeList)); + +_return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index eace144e0b4702945b5ce107c5290bd570d5e1fd..aff2c2edcdad57e162da2f005efbcc7d36be6d37 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2094,14 +2094,14 @@ TEST(multiThread, getSetRmSameDbVgroup) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); taosSsleep(1); - pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2146,14 +2146,14 @@ TEST(multiThread, getSetRmDiffDbVgroup) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); taosSsleep(1); - pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2198,13 +2198,13 @@ TEST(multiThread, ctableMeta) { strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); taosSsleep(1); - pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); + taosThreadCreate(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); while (true) { if (ctgTestDeadLoop) { diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 8acb6f7e8d933153c4d35437529dfaf46acf6174..85356a862ce282ac53aaad4ee72f0a77b19f115c 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -29,7 +29,7 @@ struct SDataSinkHandle; typedef struct SDataSinkManager { SDataSinkMgtCfg cfg; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index b34067ba4e7e19e70a8033f6336b9e46d5f4086e..bcbfeb7015390bc8ddadd08349cd858e93a9bdc1 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -50,7 +50,7 @@ typedef struct SGroupResInfo { int32_t totalGroup; int32_t currentGroup; int32_t index; - SArray* pRows; // SArray + SArray* pRows; // SArray bool ordered; int32_t position; } SGroupResInfo; @@ -67,10 +67,15 @@ typedef struct SResultRow { char *key; // start key of current result row } SResultRow; +typedef struct SResultRowPosition { + int32_t pageId; + int32_t offset; +} SResultRowPosition; + typedef struct SResultRowInfo { - SList* pRows; - SResultRow** pResult; // result list -// int16_t type:8; // data type for hash key + SList *pRows; + SResultRowPosition *pPosition; + SResultRow **pResult; // result list int32_t size; // number of result set int32_t capacity; // max capacity int32_t curPos; // current active result row index of pResult list @@ -131,7 +136,7 @@ static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffs assert(rowOffset >= 0); int32_t numOfRows = 1;//(int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); - return ((char *)page->data) + rowOffset + offset * numOfRows; + return (char*) page + rowOffset + offset * numOfRows; } //bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); @@ -139,12 +144,7 @@ static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffs __filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr); -SResultRowPool* initResultRowPool(size_t size); SResultRow* getNewResultRow(SResultRowPool* p); -int64_t getResultRowPoolMemSize(SResultRowPool* p); -void* destroyResultRowPool(SResultRowPool* p); -int32_t getNumOfAllocatedResultRows(SResultRowPool* p); -int32_t getNumOfUsedResultRows(SResultRowPool* p); typedef struct { SArray* pResult; // SArray diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index b7006ed8ad16684a3e3ff20992a8c261bcfb353e..dc3313e23e738b6ff98b1c287b223c8dae3a55bd 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -240,12 +240,12 @@ typedef struct STaskAttr { SArray* pUdfInfo; // no need to free } STaskAttr; -typedef int32_t (*__optr_open_fn_t)(void* param); -typedef SSDataBlock* (*__optr_fn_t)(void* param, bool* newgroup); -typedef void (*__optr_close_fn_t)(void* param, int32_t num); - struct SOperatorInfo; +typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* param); +typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* param, bool* newgroup); +typedef void (*__optr_close_fn_t)(void* param, int32_t num); + typedef struct STaskIdInfo { uint64_t queryId; // this is also a request id uint64_t subplanId; @@ -269,36 +269,36 @@ typedef struct SExecTaskInfo { } SExecTaskInfo; typedef struct STaskRuntimeEnv { - jmp_buf env; - STaskAttr* pQueryAttr; - uint32_t status; // query status - void* qinfo; - uint8_t scanFlag; // denotes reversed scan of data or not - void* pTsdbReadHandle; - - int32_t prevGroupId; // previous executed group id - bool enableGroupData; - SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - SHashObj* pResultRowHashTable; // quick locate the window object for each result - SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not - SArray* pResultRowArrayList; // The array list that contains the Result rows - char* keyBuf; // window key buffer + jmp_buf env; + STaskAttr* pQueryAttr; + uint32_t status; // query status + void* qinfo; + uint8_t scanFlag; // denotes reversed scan of data or not + void* pTsdbReadHandle; + + int32_t prevGroupId; // previous executed group id + bool enableGroupData; + SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not + SArray* pResultRowArrayList; // The array list that contains the Result rows + char* keyBuf; // window key buffer // The window result objects pool, all the resultRow Objects are allocated and managed by this object. - char** prevRow; + char** prevRow; SResultRowPool* pool; - SArray* prevResult; // intermediate result, SArray - STSBuf* pTsBuf; // timestamp filter list - STSCursor cur; + SArray* prevResult; // intermediate result, SArray + STSBuf* pTsBuf; // timestamp filter list + STSCursor cur; - char* tagVal; // tag value of current data block + char* tagVal; // tag value of current data block struct SScalarFunctionSupport* scalarSup; SSDataBlock* outputBuf; STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure struct SOperatorInfo* proot; - SGroupResInfo groupResInfo; - int64_t currentOffset; // dynamic offset value + SGroupResInfo groupResInfo; + int64_t currentOffset; // dynamic offset value STableQueryInfo* current; SRspResultInfo resultInfo; @@ -310,6 +310,8 @@ enum { OP_IN_EXECUTING = 1, OP_RES_TO_RETURN = 2, OP_EXEC_DONE = 3, + OP_OPENED = 4, + OP_NOT_OPENED = 5 }; typedef struct SOperatorInfo { @@ -320,14 +322,16 @@ typedef struct SOperatorInfo { char* name; // name, used to show the query execution plan void* info; // extension attribution SExprInfo* pExpr; - STaskRuntimeEnv* pRuntimeEnv; // todo remove it + STaskRuntimeEnv* pRuntimeEnv; // todo remove it SExecTaskInfo* pTaskInfo; + SOperatorCostInfo cost; struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - __optr_open_fn_t openFn; - __optr_fn_t nextDataFn; + __optr_fn_t getNextFn; + __optr_fn_t cleanupFn; __optr_close_fn_t closeFn; + __optr_open_fn_t _openFn; // DO NOT invoke this function directly } SOperatorInfo; typedef struct { @@ -346,7 +350,7 @@ typedef struct SQInfo { STaskAttr query; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; - pthread_mutex_t lock; // used to synchronize the rsp/query threads + TdThreadMutex lock; // used to synchronize the rsp/query threads tsem_t ready; int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context @@ -452,45 +456,44 @@ typedef struct SSysTableScanInfo { } SSysTableScanInfo; typedef struct SOptrBasicInfo { - SResultRowInfo resultRowInfo; - int32_t* rowCellInfoOffset; // offset value for each row result cell info - SqlFunctionCtx* pCtx; - SSDataBlock* pRes; - int32_t capacity; + SResultRowInfo resultRowInfo; + int32_t* rowCellInfoOffset; // offset value for each row result cell info + SqlFunctionCtx* pCtx; + SSDataBlock* pRes; + int32_t capacity; } SOptrBasicInfo; //TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset typedef struct SAggSupporter { - SHashObj* pResultRowHashTable; // quick locate the window object for each result - SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not - SArray* pResultRowArrayList; // The array list that contains the Result rows - char* keyBuf; // window key buffer - SResultRowPool *pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not + SArray* pResultRowArrayList; // The array list that contains the Result rows + char* keyBuf; // window key buffer + SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; typedef struct STableIntervalOperatorInfo { - SOptrBasicInfo binfo; - SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file - SGroupResInfo groupResInfo; - SInterval interval; - STimeWindow win; - int32_t precision; - bool timeWindowInterpo; - char **pRow; - SAggSupporter aggSup; - STableQueryInfo *pCurrent; - int32_t order; + SOptrBasicInfo binfo; + SGroupResInfo groupResInfo; + SInterval interval; + STimeWindow win; + int32_t precision; + bool timeWindowInterpo; + char **pRow; + SAggSupporter aggSup; + STableQueryInfo *pCurrent; + int32_t order; } STableIntervalOperatorInfo; typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file - SAggSupporter aggSup; - STableQueryInfo *current; - uint32_t groupId; - SGroupResInfo groupResInfo; - STableQueryInfo *pTableQueryInfo; + SOptrBasicInfo binfo; + SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file + SAggSupporter aggSup; + STableQueryInfo *current; + uint32_t groupId; + SGroupResInfo groupResInfo; + STableQueryInfo *pTableQueryInfo; } SAggOperatorInfo; typedef struct SProjectOperatorInfo { @@ -500,28 +503,27 @@ typedef struct SProjectOperatorInfo { } SProjectOperatorInfo; typedef struct SLimitOperatorInfo { - int64_t limit; - int64_t total; + SLimit limit; + int64_t currentOffset; + int64_t currentRows; } SLimitOperatorInfo; typedef struct SSLimitOperatorInfo { - int64_t groupTotal; - int64_t currentGroupOffset; - - int64_t rowsTotal; - int64_t currentOffset; - SLimit limit; - SLimit slimit; - - char** prevRow; - SArray* orderColumnList; - bool hasPrev; - bool ignoreCurrentGroup; - bool multigroupResult; - SSDataBlock* pRes; // result buffer - SSDataBlock* pPrevBlock; - int64_t capacity; - int64_t threshold; + int64_t groupTotal; + int64_t currentGroupOffset; + int64_t rowsTotal; + int64_t currentOffset; + SLimit limit; + SLimit slimit; + char** prevRow; + SArray* orderColumnList; + bool hasPrev; + bool ignoreCurrentGroup; + bool multigroupResult; + SSDataBlock* pRes; // result buffer + SSDataBlock* pPrevBlock; + int64_t capacity; + int64_t threshold; } SSLimitOperatorInfo; typedef struct SFilterOperatorInfo { @@ -544,14 +546,15 @@ typedef struct SGroupbyOperatorInfo { char* prevData; // previous group by value } SGroupbyOperatorInfo; -typedef struct SSWindowOperatorInfo { +typedef struct SSessionAggOperatorInfo { SOptrBasicInfo binfo; + SAggSupporter aggSup; STimeWindow curWindow; // current time window TSKEY prevTs; // previous timestamp int32_t numOfRows; // number of rows int32_t start; // start row index bool reptScan; // next round scan -} SSWindowOperatorInfo; +} SSessionAggOperatorInfo; typedef struct SStateWindowOperatorInfo { SOptrBasicInfo binfo; @@ -563,23 +566,6 @@ typedef struct SStateWindowOperatorInfo { bool reptScan; } SStateWindowOperatorInfo; -typedef struct SDistinctDataInfo { - int32_t index; - int32_t type; - int32_t bytes; -} SDistinctDataInfo; - -typedef struct SDistinctOperatorInfo { - SHashObj* pSet; - SSDataBlock* pRes; - bool recordNullVal; // has already record the null value, no need to try again - int64_t threshold; - int64_t outputCapacity; - int32_t totalBytes; - char* buf; - SArray* pDistinctDataInfo; -} SDistinctOperatorInfo; - typedef struct SSortedMergeOperatorInfo { SOptrBasicInfo binfo; bool hasVarCol; @@ -605,41 +591,61 @@ typedef struct SSortedMergeOperatorInfo { } SSortedMergeOperatorInfo; typedef struct SOrderOperatorInfo { - uint32_t sortBufSize; // max buffer size for in-memory sort - SSDataBlock *pDataBlock; - bool hasVarCol; // has variable length column, such as binary/varchar/nchar - SArray *orderInfo; - bool nullFirst; - SSortHandle *pSortHandle; - - int32_t bufPageSize; - int32_t numOfRowsInRes; + uint32_t sortBufSize; // max buffer size for in-memory sort + SSDataBlock *pDataBlock; + bool hasVarCol; // has variable length column, such as binary/varchar/nchar + SArray *orderInfo; + bool nullFirst; + SSortHandle *pSortHandle; + int32_t bufPageSize; + int32_t numOfRowsInRes; // TODO extact struct - int64_t startTs; // sort start time - uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - uint64_t totalSize; // total load bytes from remote - uint64_t totalRows; // total number of rows - uint64_t totalElapsed; // total elapsed time + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + uint64_t totalSize; // total load bytes from remote + uint64_t totalRows; // total number of rows + uint64_t totalElapsed; // total elapsed time } SOrderOperatorInfo; +typedef struct SDistinctDataInfo { + int32_t index; + int32_t type; + int32_t bytes; +} SDistinctDataInfo; + +typedef struct SDistinctOperatorInfo { + SHashObj* pSet; + SSDataBlock* pRes; + bool recordNullVal; // has already record the null value, no need to try again + int64_t threshold; + int64_t outputCapacity; + int32_t totalBytes; + char* buf; + SArray* pDistinctDataInfo; +} SDistinctOperatorInfo; + SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); -SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, SNode* pCondition, SEpSet epset, SArray* colList, - SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, + SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); +SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, + SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfDownstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, + const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); + SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, @@ -665,8 +671,6 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); // SSDataBlock* doSLimit(void* param, bool* newgroup); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); @@ -678,9 +682,9 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, - int32_t* rowCellInfoOffset); -void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity, int32_t numOfInputRows); + +void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); + void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 893138504909d69a2bc9ebba1958677007b2f00f..4c13820bca4aa68b42f7c0063974644bd2b000d4 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -44,7 +44,7 @@ typedef struct SDataDispatchHandle { int32_t status; bool queryEnd; uint64_t useconds; - pthread_mutex_t mutex; + TdThreadMutex mutex; } SDataDispatchHandle; static bool needCompress(const SSDataBlock* pData, const SDataBlockDescNode* pSchema) { @@ -119,7 +119,8 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, return false; } - pBuf->allocSize = sizeof(SRetrieveTableRsp) + pDispatcher->pSchema->resultRowSize * pInput->pData->info.rows; + // struct size + data payload + length for each column + pBuf->allocSize = sizeof(SRetrieveTableRsp) + pDispatcher->pSchema->resultRowSize * pInput->pData->info.rows + pInput->pData->info.numOfCols * sizeof(int32_t); pBuf->pData = malloc(pBuf->allocSize); if (pBuf->pData == NULL) { qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); @@ -129,19 +130,19 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, } static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); int32_t blockNums = taosQueueSize(pDispatcher->pDataBlocks); int32_t status = (0 == blockNums ? DS_BUF_EMPTY : (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); pDispatcher->status = status; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return status; } static int32_t getStatus(SDataDispatchHandle* pDispatcher) { - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); int32_t status = pDispatcher->status; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return status; } @@ -159,10 +160,10 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); pDispatcher->queryEnd = true; pDispatcher->useconds = useconds; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); } static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { @@ -194,11 +195,11 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->compressed = pEntry->compressed; tfree(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); - pthread_mutex_lock(&pDispatcher->mutex); + taosThreadMutexLock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->pSchema->precision; - pthread_mutex_unlock(&pDispatcher->mutex); + taosThreadMutexUnlock(&pDispatcher->mutex); return TSDB_CODE_SUCCESS; } @@ -212,7 +213,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { taosFreeQitem(pBuf); } taosCloseQueue(pDispatcher->pDataBlocks); - pthread_mutex_destroy(&pDispatcher->mutex); + taosThreadMutexDestroy(&pDispatcher->mutex); } int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { @@ -231,7 +232,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->status = DS_BUF_EMPTY; dispatcher->queryEnd = false; dispatcher->pDataBlocks = taosOpenQueue(); - pthread_mutex_init(&dispatcher->mutex, NULL); + taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 343b3a3c95080ce82df9ca6db25eed2e9bc2125d..997397314fc253d55530d9729c3961cc1357950b 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -22,7 +22,8 @@ static SDataSinkManager gDataSinkManager = {0}; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg) { gDataSinkManager.cfg = *cfg; - pthread_mutex_init(&gDataSinkManager.mutex, NULL); + taosThreadMutexInit(&gDataSinkManager.mutex, NULL); + return 0; // to avoid compiler eror } int32_t dsCreateDataSinker(const SDataSinkNode *pDataSink, DataSinkHandle* pHandle) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 9d77e23d388aea299b2974db58091dfb5a5cbea9..a04a10ef95242ef59b74001a1ea07d7cc004fd24 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -59,7 +59,8 @@ int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->capacity = size; pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); - if (pResultRowInfo->pResult == NULL) { + pResultRowInfo->pPosition = calloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); + if (pResultRowInfo->pResult == NULL || pResultRowInfo->pPosition == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -182,22 +183,6 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return rowSize; } -SResultRowPool* initResultRowPool(size_t size) { - SResultRowPool* p = calloc(1, sizeof(SResultRowPool)); - if (p == NULL) { - return NULL; - } - - p->numOfElemPerBlock = 128; - - p->elemSize = (int32_t) size; - p->blockSize = p->numOfElemPerBlock * p->elemSize; - p->position.pos = 0; - - p->pData = taosArrayInit(8, POINTER_BYTES); - return p; -} - SResultRow* getNewResultRow(SResultRowPool* p) { if (p == NULL) { return NULL; @@ -221,132 +206,6 @@ SResultRow* getNewResultRow(SResultRowPool* p) { return ptr; } -int64_t getResultRowPoolMemSize(SResultRowPool* p) { - if (p == NULL) { - return 0; - } - - return taosArrayGetSize(p->pData) * p->blockSize; -} - -int32_t getNumOfAllocatedResultRows(SResultRowPool* p) { - return (int32_t) taosArrayGetSize(p->pData) * p->numOfElemPerBlock; -} - -int32_t getNumOfUsedResultRows(SResultRowPool* p) { - return getNumOfAllocatedResultRows(p) - p->numOfElemPerBlock + p->position.pos; -} - -void* destroyResultRowPool(SResultRowPool* p) { - if (p == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(p->pData); - for(int32_t i = 0; i < size; ++i) { - void** ptr = taosArrayGet(p->pData, i); - tfree(*ptr); - } - - taosArrayDestroy(p->pData); - - tfree(p); - return NULL; -} - -void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen) { - uint32_t numOfGroup = (uint32_t) taosArrayGetSize(pRes); - tbufWriteUint32(bw, numOfGroup); - tbufWriteUint16(bw, tagLen); - - for(int32_t i = 0; i < numOfGroup; ++i) { - SInterResult* pOne = taosArrayGet(pRes, i); - if (tagLen > 0) { - tbufWriteBinary(bw, pOne->tags, tagLen); - } - - uint32_t numOfCols = (uint32_t) taosArrayGetSize(pOne->pResult); - tbufWriteUint32(bw, numOfCols); - for(int32_t j = 0; j < numOfCols; ++j) { - SStddevInterResult* p = taosArrayGet(pOne->pResult, j); - uint32_t numOfRows = (uint32_t) taosArrayGetSize(p->pResult); - - tbufWriteUint16(bw, p->colId); - tbufWriteUint32(bw, numOfRows); - - for(int32_t k = 0; k < numOfRows; ++k) { -// SResPair v = *(SResPair*) taosArrayGet(p->pResult, k); -// tbufWriteDouble(bw, v.avg); -// tbufWriteInt64(bw, v.key); - } - } - } -} - -SArray* interResFromBinary(const char* data, int32_t len) { - SBufferReader br = tbufInitReader(data, len, false); - uint32_t numOfGroup = tbufReadUint32(&br); - uint16_t tagLen = tbufReadUint16(&br); - - char* tag = NULL; - if (tagLen > 0) { - tag = calloc(1, tagLen); - } - - SArray* pResult = taosArrayInit(4, sizeof(SInterResult)); - - for(int32_t i = 0; i < numOfGroup; ++i) { - if (tagLen > 0) { - memset(tag, 0, tagLen); - tbufReadToBinary(&br, tag, tagLen); - } - - uint32_t numOfCols = tbufReadUint32(&br); - - SArray* p = taosArrayInit(numOfCols, sizeof(SStddevInterResult)); - for(int32_t j = 0; j < numOfCols; ++j) { -// int16_t colId = tbufReadUint16(&br); - int32_t numOfRows = tbufReadUint32(&br); - -// SStddevInterResult interRes = {.colId = colId, .pResult = taosArrayInit(4, sizeof(struct SResPair)),}; - for(int32_t k = 0; k < numOfRows; ++k) { -// SResPair px = {0}; -// px.avg = tbufReadDouble(&br); -// px.key = tbufReadInt64(&br); -// -// taosArrayPush(interRes.pResult, &px); - } - -// taosArrayPush(p, &interRes); - } - - char* p1 = NULL; - if (tagLen > 0) { - p1 = malloc(tagLen); - memcpy(p1, tag, tagLen); - } - - SInterResult d = {.pResult = p, .tags = p1,}; - taosArrayPush(pResult, &d); - } - - tfree(tag); - return pResult; -} - -void freeInterResult(void* param) { - SInterResult* pResult = (SInterResult*) param; - tfree(pResult->tags); - - int32_t numOfCols = (int32_t) taosArrayGetSize(pResult->pResult); - for(int32_t i = 0; i < numOfCols; ++i) { - SStddevInterResult *p = taosArrayGet(pResult->pResult, i); - taosArrayDestroy(p->pResult); - } - - taosArrayDestroy(pResult->pResult); -} - void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); @@ -360,7 +219,7 @@ void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) taosArrayDestroy(pGroupResInfo->pRows); } - pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pResult, pResultInfo->size, POINTER_BYTES); + pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pPosition, pResultInfo->size, sizeof(SResultRowPosition)); pGroupResInfo->index = 0; assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index fabaa2d31dc822c7d926fc674bbae37d783ba7f8..cc9921ce73c655508a395733a43170d78c0ebf3a 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -30,7 +30,7 @@ #include "query.h" typedef struct STaskMgmt { - pthread_mutex_t lock; + TdThreadMutex lock; SCacheObj *qinfoPool; // query handle pool int32_t vgId; bool closed; @@ -158,7 +158,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { int64_t st = 0; st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->nextDataFn(pTaskInfo->pRoot, &newgroup); + *pRes = pTaskInfo->pRoot->getNextFn(pTaskInfo->pRoot, &newgroup); uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3f79c02cb001cd30c56f66944f5b100c9e3e9536..d4e9f47f8576f9997e522820ff933df7d23cea74 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -28,15 +28,16 @@ #include "tsort.h" #include "ttime.h" +#include "../../function/inc/taggfunction.h" #include "executorimpl.h" #include "function.h" +#include "query.h" #include "tcompare.h" #include "tcompression.h" #include "thash.h" +#include "tsdb.h" #include "ttypes.h" -#include "query.h" #include "vnode.h" -#include "tsdb.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -213,6 +214,11 @@ static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); + +static void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput); +static void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput); +static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput); + static void destroyOperatorInfo(SOperatorInfo* pOperator); static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput); @@ -223,7 +229,17 @@ static void doSetOperatorCompleted(SOperatorInfo* pOperator) { } } -static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity); +#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) +#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) + +static int32_t operatorDummyOpenFn(SOperatorInfo *pOperator) { + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +static void operatorDummyCloseFn(void* param, int32_t numOfCols) {} + +static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset); static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); @@ -432,10 +448,12 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + pResultRowInfo->pPosition = realloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); pResultRowInfo->pResult = (SResultRow **)t; int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); + memset(&pResultRowInfo->pPosition[pResultRowInfo->capacity], 0, sizeof(SResultRowPosition)); pResultRowInfo->capacity = (int32_t)newCapacity; } @@ -552,7 +570,47 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR return pResultRowInfo->pResult[pResultRowInfo->curPos]; } -static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, +SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) { + SFilePage *pData = NULL; + + // in the first scan, new space needed for results + int32_t pageId = -1; + SIDList list = getDataBufPagesIdList(pResultBuf, tableGroupId); + + if (taosArrayGetSize(list) == 0) { + pData = getNewBufPage(pResultBuf, tableGroupId, &pageId); + pData->num = sizeof(SFilePage); + } else { + SPageInfo* pi = getLastPageInfo(list); + pData = getBufPage(pResultBuf, getPageId(pi)); + pageId = getPageId(pi); + + if (pData->num + interBufSize + sizeof(SResultRow) > getBufPageSize(pResultBuf)) { + // release current page first, and prepare the next one + releaseBufPageInfo(pResultBuf, pi); + + pData = getNewBufPage(pResultBuf, tableGroupId, &pageId); + if (pData != NULL) { + pData->num = sizeof(SFilePage); + } + } + } + + if (pData == NULL) { + return NULL; + } + + // set the number of rows in current disk page + SResultRow* pResultRow = (SResultRow*)((char*)pData + pData->num); + pResultRow->pageId = pageId; + pResultRow->offset = (int32_t)pData->num; + + pData->num += interBufSize + sizeof(SResultRow); + + return pResultRow; +} + +static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { bool existed = false; SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, tableGroupId); @@ -596,7 +654,7 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int SResultRow *pResult = NULL; if (p1 == NULL) { - pResult = getNewResultRow(pSup->pool); + pResult = getNewResultRow_rv(pResultBuf, tableGroupId, pSup->resultRowSize); int32_t ret = initResultRow(pResult); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -611,6 +669,7 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int } pResultRowInfo->curPos = pResultRowInfo->size; + pResultRowInfo->pPosition[pResultRowInfo->size] = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; int64_t index = pResultRowInfo->curPos; @@ -730,6 +789,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedBuf *pRes if (taosArrayGetSize(list) == 0) { pData = getNewBufPage(pResultBuf, tid, &pageId); + pData->num = sizeof(SFilePage); } else { SPageInfo* pi = getLastPageInfo(list); pData = getBufPage(pResultBuf, getPageId(pi)); @@ -738,9 +798,10 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedBuf *pRes if (pData->num + size > getBufPageSize(pResultBuf)) { // release current page first, and prepare the next one releaseBufPageInfo(pResultBuf, pi); + pData = getNewBufPage(pResultBuf, tid, &pageId); if (pData != NULL) { - assert(pData->num == 0); // number of elements must be 0 for new allocated buffer + pData->num = sizeof(SFilePage); } } } @@ -800,9 +861,9 @@ static void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pR static int32_t setResultOutputBufByKey_rv(SResultRowInfo *pResultRowInfo, int64_t id, STimeWindow *win, bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset, SDiskbasedBuf *pBuf, SAggSupporter *pAggSup, SExecTaskInfo* pTaskInfo) { + int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter *pAggSup, SExecTaskInfo* pTaskInfo) { assert(win->skey <= win->ekey); - SResultRow *pResultRow = doSetResultOutBufByKey_rv(pResultRowInfo, id, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, + SResultRow *pResultRow = doSetResultOutBufByKey_rv(pAggSup->pResultBuf, pResultRowInfo, id, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup); if (pResultRow == NULL) { @@ -810,19 +871,10 @@ static int32_t setResultOutputBufByKey_rv(SResultRowInfo *pResultRowInfo, int64_ return TSDB_CODE_SUCCESS; } - // not assign result buffer yet, add new result buffer - if (pResultRow->pageId == -1) { // todo intermediate result size - int32_t ret = addNewWindowResultBuf(pResultRow, pBuf, (int32_t) tableGroupId, 0); - if (ret != TSDB_CODE_SUCCESS) { - return -1; - } - } - // set time window for current result pResultRow->win = (*win); *pResult = pResultRow; - setResultRowOutputBufInitCtx_rv(pBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); - + setResultRowOutputBufInitCtx_rv(pAggSup->pResultBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); return TSDB_CODE_SUCCESS; } @@ -976,20 +1028,21 @@ static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo *pDataBlockInfo, TSKEY *p } static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, - int32_t numOfTotal, int32_t numOfOutput, int32_t order) { + int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].size = forwardStep; pCtx[k].startTs = pWin->skey; // keep it temporarialy - int32_t startOffset = pCtx[k].startRow; - bool hasAgg = pCtx[k].isAggSet; + int32_t startOffset = pCtx[k].input.startRowIndex; + bool hasAgg = pCtx[k].input.colDataAggIsSet; + int32_t numOfRows = pCtx[k].input.numOfRows; int32_t pos = (order == TSDB_ORDER_ASC) ? offset : offset - (forwardStep - 1); - pCtx[k].startRow = pos; + pCtx[k].input.startRowIndex = pos; + pCtx[k].input.numOfRows = forwardStep; if (tsCol != NULL) { - pCtx[k].ptsList = &tsCol[pos]; + pCtx[k].ptsList = tsCol; } // not a whole block involved in query processing, statistics data can not be used @@ -999,12 +1052,13 @@ static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t of } if (functionNeedToExecute(&pCtx[k])) { -// pCtx[k].fpSet.process(&pCtx[k]); + pCtx[k].fpSet.process(&pCtx[k]); } // restore it - pCtx[k].isAggSet = hasAgg; - pCtx[k].startRow = startOffset; + pCtx[k].input.colDataAggIsSet = hasAgg; + pCtx[k].input.startRowIndex = startOffset; + pCtx[k].input.numOfRows = numOfRows; } } @@ -1422,8 +1476,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); tsCols = (int64_t*) pColDataInfo->pData; - assert(tsCols[0] == pSDataBlock->info.window.skey && - tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); + assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); } int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); @@ -1434,7 +1487,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul SResultRow* pResult = NULL; int32_t ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1456,7 +1509,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow w = pRes->win; ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); + tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1474,7 +1527,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul // restore current time window ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1494,7 +1547,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul // null data, failed to allocate more memory buffer int32_t code = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1515,7 +1568,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); } - static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; @@ -1675,7 +1727,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn tfree(pInfo->prevData); } -static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { +static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->current; @@ -2012,9 +2064,7 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI return pFuncCtx; } -static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { - size_t numOfOutput = taosArrayGetSize(pExprInfo); - +static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset) { SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; @@ -2027,7 +2077,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC } for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + SExprInfo* pExpr = &pExprInfo[i]; SExprBasicInfo *pFunct = &pExpr->base; SqlFunctionCtx* pCtx = &pFuncCtx[i]; @@ -2845,6 +2895,8 @@ int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; + *status = BLK_DATA_ALL_NEEDED; + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); if (pBlock->pDataBlock == NULL) { return terrno; @@ -3261,10 +3313,9 @@ void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } +// TODO fix this bug. int32_t initResultRow(SResultRow *pResultRow) { pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); - pResultRow->pageId = -1; - pResultRow->offset = -1; return TSDB_CODE_SUCCESS; } @@ -3320,7 +3371,7 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t int64_t tid = 0; int64_t groupId = 0; - SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup); + SResultRow* pRow = doSetResultOutBufByKey_rv(pSup->pResultBuf, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -3384,11 +3435,11 @@ void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) int32_t functionId = pCtx[i].functionId; if (functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { needCopyTs = true; - if (i > 0 && pCtx[i-1].functionId == FUNCTION_TS_DUMMY){ + if (i > 0 && pCtx[i-1].functionId == FUNCTION_TS_DUMMY) { SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data src = pColRes->pData; } - }else if(functionId == FUNCTION_TS_DUMMY) { + } else if(functionId == FUNCTION_TS_DUMMY) { tsNum++; } } @@ -3461,48 +3512,44 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCt pTableScanInfo->reverseTimes = 0; } -void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - int32_t numOfOutput = pOperator->numOfOutput; -// if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { -// // for each group result, call the finalize function for each column -// if (pQueryAttr->groupbyColumn) { -// closeAllResultRows(pResultRowInfo); -// } -// -// for (int32_t i = 0; i < pResultRowInfo->size; ++i) { -// SResultRow *buf = pResultRowInfo->pResult[i]; -// if (!isResultRowClosed(pResultRowInfo, i)) { -// continue; -// } -// -// setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); -// -// for (int32_t j = 0; j < numOfOutput; ++j) { -//// pCtx[j].startTs = buf->win.skey; -//// if (pCtx[j].functionId < 0) { -//// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); -//// } else { -//// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); -//// } -// } -// -// -// /* -// * set the number of output results for group by normal columns, the number of output rows usually is 1 except -// * the top and bottom query -// */ -// buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); -// } -// -// } else { +void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t j = 0; j < numOfOutput; ++j) { + pCtx[j].fpSet.finalize(&pCtx[j]); + } +} + +void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf *pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRowPosition* pPos = &pResultRowInfo->pPosition[i]; + + SFilePage* bufPage = getBufPage(pBuf, pPos->pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset); + if (!isResultRowClosed(pResultRowInfo, i)) { + continue; + } + for (int32_t j = 0; j < numOfOutput; ++j) { -// if (pCtx[j].functionId < 0) { -// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); -// } else { - pCtx[j].fpSet.finalize(&pCtx[j]); -// } + pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); + + struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; + if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { + continue; + } + + pCtx[j].fpSet.finalize(&pCtx[j]); + + if (pRow->numOfRows < pResInfo->numOfRes) { + pRow->numOfRows = pResInfo->numOfRes; + } } -// } + + releaseBufPage(pBuf, bufPage); + /* + * set the number of output results for group by normal columns, the number of output rows usually is 1 except + * the top and bottom query + */ +// buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); + } } static bool hasMainOutput(STaskAttr *pQueryAttr) { @@ -3597,31 +3644,29 @@ void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage* bufPage = getBufPage(pBuf, pResult->pageId); - int32_t offset = 0; +// int32_t offset = 0; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { - offset += pCtx[i].resDataInfo.bytes; +// offset += pCtx[i].resDataInfo.bytes; continue; } - pCtx[i].pOutput = getPosInResultPage_rv(bufPage, pResult->offset, offset); - offset += pCtx[i].resDataInfo.bytes; +// offset += pCtx[i].resDataInfo.bytes; - int32_t functionId = pCtx[i].functionId; - if (functionId < 0) { - continue; - } +// int32_t functionId = pCtx[i].functionId; +// if (functionId < 0) { +// continue; +// } +// if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { +// if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput; +// } - if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { - if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput; + if (!pResInfo->initialized) { + pCtx[i].fpSet.init(&pCtx[i], pResInfo); } - - // if (!pResInfo->initialized) { - // aAggs[functionId].init(&pCtx[i], pResInfo); - // } } } @@ -3635,7 +3680,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, i int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; SResultRow* pResultRow = - doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid, pTaskInfo, false, &pAggInfo->aggSup); + doSetResultOutBufByKey_rv(pAggInfo->pResultBuf, pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid, pTaskInfo, false, &pAggInfo->aggSup); assert (pResultRow != NULL); /* @@ -3876,8 +3921,7 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { * @param pQInfo * @param result */ - -static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity) { +static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -3895,13 +3939,19 @@ static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResI step = -1; } + int32_t nrows = pBlock->info.rows; + for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { - SResultRow* pRow = taosArrayGetP(pGroupResInfo->pRows, i); + SResultRowPosition* pPos = taosArrayGet(pGroupResInfo->pRows, i); + SFilePage *page = getBufPage(pBuf, pPos->pageId); + + SResultRow* pRow = (SResultRow*)((char*)page + pPos->offset); if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; continue; } + // TODO copy multiple rows? int32_t numOfRowsToCopy = pRow->numOfRows; if (numOfResult + numOfRowsToCopy >= rowCapacity) { break; @@ -3909,20 +3959,17 @@ static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResI pGroupResInfo->index += 1; - SFilePage *page = getBufPage(pBuf, pRow->pageId); - - int32_t offset = 0; for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); - int32_t bytes = pColInfoData->info.bytes; - - char *out = pColInfoData->pData + numOfResult * bytes; - char *in = getPosInResultPage_rv(page, pRow->offset, offset); - memcpy(out, in, bytes * numOfRowsToCopy); + SResultRowEntryInfo* pEntryInfo = getResultCell(pRow, j, rowCellOffset); - offset += bytes; + char* in = GET_ROWCELL_INTERBUF(pEntryInfo); + colDataAppend(pColInfoData, nrows, in, pEntryInfo->numOfRes == 0); } + releaseBufPage(pBuf, page); + nrows += 1; + numOfResult += numOfRowsToCopy; if (numOfResult == rowCapacity) { // output buffer is full break; @@ -3934,16 +3981,16 @@ static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResI return 0; } -static void toSDatablock(SGroupResInfo *pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity) { +static void toSDatablock(SGroupResInfo *pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - pBlock->info.rows = 0; + blockDataClearup(pBlock); if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { return; } int32_t orderType = TSDB_ORDER_ASC;//(pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - doCopyToSDataBlock(pBuf, pGroupResInfo, orderType, pBlock, rowCapacity); + doCopyToSDataBlock(pBuf, pGroupResInfo, orderType, pBlock, rowCapacity, rowCellOffset); // add condition (pBlock->info.rows >= 1) just to runtime happy blockDataUpdateTsWindow(pBlock); @@ -4653,9 +4700,7 @@ static void doCloseAllTimeWindow(STaskRuntimeEnv* pRuntimeEnv) { } } -static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; - +static SSDataBlock* doTableScanImpl(SOperatorInfo *pOperator, bool* newgroup) { STableScanInfo *pTableScanInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; @@ -4685,7 +4730,7 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { // } // this function never returns error? - uint32_t status; + uint32_t status = BLK_DATA_ALL_NEEDED; int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { @@ -4703,9 +4748,7 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { return NULL; } -static SSDataBlock* doTableScan(void* param, bool *newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - +static SSDataBlock* doTableScan(SOperatorInfo *pOperator, bool *newgroup) { STableScanInfo *pTableScanInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; @@ -4771,8 +4814,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { return p; } -static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*)param; +static SSDataBlock* doBlockInfoScan(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -4819,9 +4861,7 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { #endif } -static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*)param; - +static SSDataBlock* doStreamBlockScan(SOperatorInfo *pOperator, bool* newgroup) { // NOTE: this operator never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; @@ -5103,6 +5143,31 @@ static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo *pOperator) { return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); } +static int32_t prepareConcurrentlyLoad(SOperatorInfo *pOperator) { + SExchangeInfo *pExchangeInfo = pOperator->info; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + // Asynchronously send all fetch requests to all sources. + for(int32_t i = 0; i < totalSources; ++i) { + int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return code; + } + } + + int64_t endTs = taosGetTimestampUs(); + qDebug("%s send all fetch request to %"PRIzu" sources completed, elapsed:%"PRId64, GET_TASKID(pTaskInfo), totalSources, endTs - startTs); + + tsem_wait(&pExchangeInfo->ready); + pOperator->cost.openCost = taosGetTimestampUs() - startTs; + + return TSDB_CODE_SUCCESS; +} + static SSDataBlock* seqLoadRemoteData(SOperatorInfo *pOperator) { SExchangeInfo *pExchangeInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; @@ -5158,12 +5223,34 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo *pOperator) { } } -static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; +static int32_t prepareLoadRemoteData(SOperatorInfo *pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + SExchangeInfo *pExchangeInfo = pOperator->info; + if (pExchangeInfo->seqLoadData) { + // do nothing for sequentially load data + } else { + int32_t code = prepareConcurrentlyLoad(pOperator); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* doLoadRemoteData(SOperatorInfo *pOperator, bool* newgroup) { SExchangeInfo *pExchangeInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + pTaskInfo->code = pOperator->_openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; @@ -5191,9 +5278,30 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { #endif } -// TODO handle the error +static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { + pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); + if (pInfo->pSourceDataInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for(int32_t i = 0; i < numOfSources; ++i) { + SSourceDataInfo dataInfo = {0}; + dataInfo.status = EX_SOURCE_DATA_NOT_READY; + dataInfo.pEx = pInfo; + dataInfo.index = i; + + void* ret = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); + if (ret == NULL) { + taosArrayDestroy(pInfo->pSourceDataInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); + SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5205,11 +5313,9 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock size_t numOfSources = LIST_LENGTH(pSources); pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); - if (pInfo->pSources == NULL) { - tfree(pInfo); - tfree(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; + pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); + if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { + goto _error; } for(int32_t i = 0; i < numOfSources; ++i) { @@ -5217,23 +5323,9 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock taosArrayPush(pInfo->pSources, pNode); } - pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); - if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { - tfree(pInfo); - tfree(pOperator); - taosArrayDestroy(pInfo->pSources); - taosArrayDestroy(pInfo->pSourceDataInfo); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - for(int32_t i = 0; i < numOfSources; ++i) { - SSourceDataInfo dataInfo = {0}; - dataInfo.status = DATA_NOT_READY; - dataInfo.pEx = pInfo; - dataInfo.index = i; - - taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); + int32_t code = initDataSource(numOfSources, pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; } size_t size = pBlock->info.numOfCols; @@ -5248,7 +5340,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = size; - pOperator->nextDataFn = doLoadRemoteData; + pOperator->getNextFn = doLoadRemoteData; pOperator->pTaskInfo = pTaskInfo; #if 1 @@ -5275,6 +5367,16 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock #endif return pOperator; + + _error: + if (pInfo != NULL) { + destroyExchangeOperatorInfo(pInfo, numOfSources); + } + + tfree(pInfo); + tfree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; } SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { @@ -5312,7 +5414,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, tfree(pInfo); tfree(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -5328,7 +5430,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->nextDataFn = doTableScan; + pOperator->getNextFn = doTableScan; pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -5353,7 +5455,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim pOperator->info = pInfo; pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = doTableScanImpl; + pOperator->getNextFn = doTableScanImpl; return pOperator; } @@ -5377,7 +5479,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; // pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->nextDataFn = doBlockInfoScan; + pOperator->getNextFn = doBlockInfoScan; return pOperator; } @@ -5410,7 +5512,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->nextDataFn = doStreamBlockScan; + pOperator->getNextFn = doStreamBlockScan; pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -5468,9 +5570,8 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0? NULL:pInfo->pRes; } -static SSDataBlock* doSysTableScan(void* param, bool* newgroup) { +static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { // build message and send to mnode to fetch the content of system tables. - SOperatorInfo* pOperator = (SOperatorInfo*) param; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; @@ -5480,7 +5581,7 @@ static SSDataBlock* doSysTableScan(void* param, bool* newgroup) { pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); } - blockDataClearup(pInfo->pRes, true); + blockDataClearup(pInfo->pRes); SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, 1); @@ -5655,7 +5756,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->nextDataFn = doSysTableScan; + pOperator->getNextFn = doSysTableScan; pOperator->closeFn = destroySysTableScannerOperatorInfo; pOperator->pTaskInfo = pTaskInfo; @@ -5736,8 +5837,8 @@ SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { return pOrderColumns; } -static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput); -static void clearupAggSup(SAggSupporter* pAggSup); +static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput, const char* pKey); +static void cleanupAggSup(SAggSupporter* pAggSup); static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { SSortedMergeOperatorInfo* pInfo = (SSortedMergeOperatorInfo*) param; @@ -5749,7 +5850,7 @@ static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { } blockDataDestroy(pInfo->binfo.pRes); - clearupAggSup(&pInfo->aggSup); + cleanupAggSup(&pInfo->aggSup); } static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { @@ -5804,7 +5905,7 @@ static void appendOneRowToDataBlock(SSDataBlock *pBlock, STupleHandle* pTupleHan } static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, bool hasVarCol, int32_t capacity) { - blockDataClearup(pDataBlock, hasVarCol); + blockDataClearup(pDataBlock); while(1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); @@ -5825,7 +5926,7 @@ SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*) param; bool newgroup = false; - return pOperator->nextDataFn(pOperator, &newgroup); + return pOperator->getNextFn(pOperator, &newgroup); } static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) { @@ -5960,7 +6061,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { while(1) { - blockDataClearup(pDataBlock, pInfo->hasVarCol); + blockDataClearup(pDataBlock); while (1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); if (pTupleHandle == NULL) { @@ -5995,8 +6096,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { return (pInfo->binfo.pRes->info.rows > 0)? pInfo->binfo.pRes:NULL; } -static SSDataBlock* doSortedMerge(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doSortedMerge(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -6030,7 +6130,7 @@ static SSDataBlock* doSortedMerge(void* param, bool* newgroup) { return doMerge(pOperator); } -static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) { +static SArray* createBlockOrder(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal) { SArray* pOrderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); size_t numOfOrder = taosArrayGetSize(pOrderVal); @@ -6039,8 +6139,8 @@ static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) { SOrder* pOrder = taosArrayGet(pOrderVal, j); orderInfo.order = pOrder->order; - for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { - SExprInfo* pExpr = taosArrayGet(pExprInfo, i); + for (int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExpr = &pExprInfo[i]; if (pExpr->base.resSchema.colId == pOrder->col.colId) { orderInfo.colIndex = i; break; @@ -6053,7 +6153,7 @@ static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) { return pOrderInfo; } -static int32_t initGroupCol(SArray* pExprInfo, SArray* pGroupInfo, SSortedMergeOperatorInfo* pInfo) { +static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGroupInfo, SSortedMergeOperatorInfo* pInfo) { if (pGroupInfo == NULL || taosArrayGetSize(pGroupInfo) == 0) { return 0; } @@ -6069,8 +6169,8 @@ static int32_t initGroupCol(SArray* pExprInfo, SArray* pGroupInfo, SSortedMergeO size_t numOfGroupCol = taosArrayGetSize(pInfo->groupInfo); for(int32_t i = 0; i < numOfGroupCol; ++i) { SColumn* pCol = taosArrayGet(pGroupInfo, i); - for(int32_t j = 0; j < taosArrayGetSize(pExprInfo); ++j) { - SExprInfo* pe = taosArrayGet(pExprInfo, j); + for(int32_t j = 0; j < numOfCols; ++j) { + SExprInfo* pe = &pExprInfo[j]; if (pe->base.resSchema.colId == pCol->colId) { taosArrayPush(plist, pCol); taosArrayPush(pInfo->groupInfo, &j); @@ -6101,29 +6201,27 @@ static int32_t initGroupCol(SArray* pExprInfo, SArray* pGroupInfo, SSortedMergeO return TSDB_CODE_SUCCESS; } -SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo) { SSortedMergeOperatorInfo* pInfo = calloc(1, sizeof(SSortedMergeOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - int32_t numOfOutput = taosArrayGetSize(pExprInfo); - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, num, &pInfo->binfo.rowCellInfoOffset); initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); if (pInfo->binfo.pCtx == NULL || pInfo->binfo.pRes == NULL) { goto _error; } - int32_t code = doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, numOfOutput); + int32_t code = doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, num, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); - code = initGroupCol(pExprInfo, pGroupInfo, pInfo); + code = initGroupCol(pExprInfo, num, pGroupInfo, pInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -6132,7 +6230,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t // pRuntimeEnv->pQueryAttr->topBotQuery, false)); pInfo->sortBufSize = 1024 * 16; // 1MB pInfo->bufPageSize = 1024; - pInfo->orderInfo = createBlockOrder(pExprInfo, pOrderVal); + pInfo->orderInfo = createBlockOrder(pExprInfo, num, pOrderVal); pInfo->binfo.capacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, pInfo->bufPageSize); @@ -6141,12 +6239,12 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->pExpr = exprArrayDup(pExprInfo); + pOperator->numOfOutput = num; + pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->nextDataFn = doSortedMerge; - pOperator->closeFn = destroySortedMergeOperatorInfo; + pOperator->getNextFn = doSortedMerge; + pOperator->closeFn = destroySortedMergeOperatorInfo; code = appendDownstream(pOperator, downstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -6157,7 +6255,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t _error: if (pInfo != NULL) { - destroySortedMergeOperatorInfo(pInfo, numOfOutput); + destroySortedMergeOperatorInfo(pInfo, num); } tfree(pInfo); @@ -6166,8 +6264,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t return NULL; } -static SSDataBlock* doSort(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doSort(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -6200,12 +6297,12 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->hasVarCol, pInfo->numOfRowsInRes); } -SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo) { +SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo) { SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { tfree(pInfo); - + tfree(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -6214,12 +6311,10 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI pInfo->bufPageSize = 1024; pInfo->numOfRowsInRes = 1024; - pInfo->pDataBlock = createOutputBuf_rv(pExprInfo, pInfo->numOfRowsInRes); - pInfo->orderInfo = createBlockOrder(pExprInfo, pOrderVal); + pInfo->orderInfo = createBlockOrder(pExprInfo, numOfCols, pOrderVal); - for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); - if (IS_VAR_DATA_TYPE(pExpr->base.resSchema.type)) { + for(int32_t i = 0; i < numOfCols; ++i) { + if (IS_VAR_DATA_TYPE(pExprInfo[i].base.resSchema.type)) { pInfo->hasVarCol = true; break; } @@ -6227,7 +6322,7 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI if (pInfo->orderInfo == NULL || pInfo->pDataBlock == NULL) { tfree(pOperator); - destroyOrderOperatorInfo(pInfo, taosArrayGetSize(pExprInfo)); + destroyOrderOperatorInfo(pInfo, numOfCols); tfree(pInfo); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -6241,7 +6336,7 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->nextDataFn = doSort; + pOperator->getNextFn = doSort; pOperator->closeFn = destroyOrderOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -6253,46 +6348,63 @@ static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { } // this is a blocking operator -static SSDataBlock* doAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; +static int32_t doOpenAggregateOptr(SOperatorInfo *pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; } SAggOperatorInfo* pAggInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pAggInfo->binfo; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; - int32_t order = TSDB_ORDER_ASC; + int32_t order = TSDB_ORDER_ASC; SOperatorInfo* downstream = pOperator->pDownstream[0]; - while(1) { + bool newgroup = true; + while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, &newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } -// if (pAggInfo->current != NULL) { -// setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfOutput); -// } + // if (pAggInfo->current != NULL) { + // setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + // } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); doAggregateImpl(pOperator, 0, pInfo->pCtx); } - doSetOperatorCompleted(pOperator); + finalizeQueryResult(pInfo->pCtx, pOperator->numOfOutput); + + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* getAggregateResult(SOperatorInfo *pOperator, bool* newgroup) { + SAggOperatorInfo *pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + pTaskInfo->code = pOperator->_openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } - finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); getNumOfResult(pInfo->pCtx, pOperator->numOfOutput, pInfo->pRes); + doSetOperatorCompleted(pOperator); return (blockDataGetNumOfRows(pInfo->pRes) != 0)? pInfo->pRes:NULL; } -static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doMultiTableAggregate(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -6302,7 +6414,7 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(&pAggInfo->groupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity); + toSDatablock(&pAggInfo->groupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity, pAggInfo->binfo.rowCellInfoOffset); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; @@ -6317,7 +6429,7 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6351,7 +6463,7 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) { updateNumOfRowsInResultRows(pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); initGroupResInfo(&pAggInfo->groupResInfo, &pInfo->resultRowInfo); - toSDatablock(&pAggInfo->groupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity); + toSDatablock(&pAggInfo->groupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity, pAggInfo->binfo.rowCellInfoOffset); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -6360,16 +6472,16 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) { return pInfo->pRes; } -static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { - SOperatorInfo* pOperator = param; - +static SSDataBlock* doProjectOperation(SOperatorInfo *pOperator, bool* newgroup) { SProjectOperatorInfo* pProjectInfo = pOperator->info; SOptrBasicInfo *pInfo = &pProjectInfo->binfo; SSDataBlock* pRes = pInfo->pRes; - blockDataClearup(pRes, true); + + blockDataClearup(pRes); if (pProjectInfo->existDataBlock) { // TODO refactor +// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; SSDataBlock* pBlock = pProjectInfo->existDataBlock; pProjectInfo->existDataBlock = NULL; *newgroup = true; @@ -6400,7 +6512,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // The downstream exec may change the value of the newgroup, so use a local variable instead. publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6440,53 +6552,53 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } -static SSDataBlock* doLimit(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*)param; +static SSDataBlock* doLimit(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } SLimitOperatorInfo* pInfo = pOperator->info; - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; SSDataBlock* pBlock = NULL; + SOperatorInfo* pDownstream = pOperator->pDownstream[0]; + while (1) { - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pDownstream->getNextFn(pDownstream, newgroup); + publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { doSetOperatorCompleted(pOperator); return NULL; } - if (pRuntimeEnv->currentOffset == 0) { + if (pInfo->currentOffset == 0) { break; - } else if (pRuntimeEnv->currentOffset >= pBlock->info.rows) { - pRuntimeEnv->currentOffset -= pBlock->info.rows; - } else { - int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); + } else if (pInfo->currentOffset >= pBlock->info.rows) { + pInfo->currentOffset -= pBlock->info.rows; + } else { // TODO handle the data movement + int32_t remain = (int32_t)(pBlock->info.rows - pInfo->currentOffset); pBlock->info.rows = remain; for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); int16_t bytes = pColInfoData->info.bytes; - memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes); + memmove(pColInfoData->pData, pColInfoData->pData + bytes * pInfo->currentOffset, remain * bytes); } - pRuntimeEnv->currentOffset = 0; + pInfo->currentOffset = 0; break; } } - if (pInfo->total + pBlock->info.rows >= pInfo->limit) { - pBlock->info.rows = (int32_t)(pInfo->limit - pInfo->total); - pInfo->total = pInfo->limit; + if (pInfo->currentRows + pBlock->info.rows >= pInfo->limit.limit) { + pBlock->info.rows = (int32_t)(pInfo->limit.limit - pInfo->currentRows); + pInfo->currentRows = pInfo->limit.limit; doSetOperatorCompleted(pOperator); } else { - pInfo->total += pBlock->info.rows; + pInfo->currentRows += pBlock->info.rows; } return pBlock; @@ -6503,7 +6615,7 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) { while (1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); + SSDataBlock *pBlock = pOperator->pDownstream[0]->getNextFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6523,54 +6635,57 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) { return NULL; } -static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; +static int32_t doOpenIntervalAgg(SOperatorInfo *pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; } STableIntervalOperatorInfo* pInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { -// toSDatablock(pAggInfo->pGroupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->binfo.pRes; - } - -// int32_t order = pQueryAttr->order.order; -// STimeWindow win = pQueryAttr->window; + // int32_t order = pQueryAttr->order.order; + // STimeWindow win = pQueryAttr->window; + bool newgroup = false; SOperatorInfo* downstream = pOperator->pDownstream[0]; - while(1) { + while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, &newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } -// setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); } - // restore the value -// pQueryAttr->order.order = order; -// pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->binfo.resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); - toSDatablock(&pInfo->groupResInfo, pInfo->pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity); + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* doIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { + STableIntervalOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + pTaskInfo->code = pOperator->_openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -6579,8 +6694,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { return pInfo->binfo.pRes->info.rows == 0? NULL:pInfo->binfo.pRes; } -static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -6606,7 +6720,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6627,7 +6741,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->binfo.resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->binfo.pCtx, &pIntervalInfo->binfo.resultRowInfo, pIntervalInfo->binfo.rowCellInfoOffset); + finalizeQueryResult(pIntervalInfo->binfo.pCtx, pOperator->numOfOutput); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->binfo.resultRowInfo); // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); @@ -6639,8 +6753,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->binfo.pRes->info.rows == 0? NULL:pIntervalInfo->binfo.pRes; } -static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doSTableIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -6669,7 +6782,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6699,8 +6812,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->binfo.pRes; } -static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -6724,7 +6836,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6833,8 +6945,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI // pSDataBlock->info.rows, pOperator->numOfOutput); } -static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doStateWindowAgg(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -6859,7 +6970,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6879,7 +6990,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + finalizeQueryResult(pBInfo->pCtx, pOperator->numOfOutput); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); @@ -6891,68 +7002,55 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; } -static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doSessionWindowAgg(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SSWindowOperatorInfo* pWindowInfo = pOperator->info; + SSessionAggOperatorInfo* pWindowInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pBInfo->pRes->info.rows == 0/* || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)*/) { pOperator->status = OP_EXEC_DONE; } return pBInfo->pRes; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - //pQueryAttr->order.order = TSDB_ORDER_ASC; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - + int32_t order = TSDB_ORDER_ASC; SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order); doSessionWindowAggImpl(pOperator, pWindowInfo, pBlock); } // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); // setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + finalizeQueryResult(pBInfo->pCtx, pOperator->numOfOutput); - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); +// initGroupResInfo(&pBInfo->groupResInfo, &pBInfo->resultRowInfo); // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pBInfo->pRes->info.rows == 0/* || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)*/) { pOperator->status = OP_EXEC_DONE; } return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; } -static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* hashGroupbyAggregate(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -6974,7 +7072,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -6995,7 +7093,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { // setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows - finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput); } else { updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); } @@ -7041,9 +7139,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, STaskRunti } } -static SSDataBlock* doFill(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - +static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { SFillOperatorInfo *pInfo = pOperator->info; pInfo->pRes->info.rows = 0; @@ -7059,7 +7155,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); + SSDataBlock* pBlock = pOperator->pDownstream[0]->getNextFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { @@ -7149,39 +7245,50 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } -int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput) { +int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput, const char* pKey) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); - pAggSup->pool = initResultRowPool(pAggSup->resultRowSize); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL || - pAggSup->pResultRowHashTable == NULL || pAggSup->pool == NULL) { + pAggSup->pResultRowHashTable == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, 4096, 4096 * 256, pKey, "/tmp/"); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + return TSDB_CODE_SUCCESS; } -static void clearupAggSup(SAggSupporter* pAggSup) { +static void cleanupAggSup(SAggSupporter* pAggSup) { tfree(pAggSup->keyBuf); taosHashCleanup(pAggSup->pResultRowHashTable); taosHashCleanup(pAggSup->pResultRowListSet); taosArrayDestroy(pAggSup->pResultRowArrayList); - destroyResultRowPool(pAggSup->pool); + destroyDiskbasedBuf(pAggSup->pResultBuf); } -static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t numOfRows, SSDataBlock* pResultBlock, const STableGroupInfo* pTableGroupInfo) { - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = pResultBlock; - pInfo->binfo.capacity = numOfRows; +static int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, + int32_t numOfRows, SSDataBlock* pResultBlock, const char* pkey) { + pBasicInfo->pCtx = createSqlFunctionCtx_rv(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset); + pBasicInfo->pRes = pResultBlock; + pBasicInfo->capacity = numOfRows; - doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, taosArrayGetSize(pExprInfo)); - pInfo->pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, pkey); +} + +static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInfo) { + STableQueryInfo* pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + if (pTableQueryInfo == NULL) { + return NULL; + } int32_t index = 0; for(int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pGroupList); ++i) { @@ -7189,7 +7296,7 @@ static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t n for(int32_t j = 0; j < taosArrayGetSize(pa); ++j) { STableKeyInfo* pk = taosArrayGet(pa, j); - STableQueryInfo* pTQueryInfo = &pInfo->pTableQueryInfo[index++]; + STableQueryInfo* pTQueryInfo = &pTableQueryInfo[index++]; pTQueryInfo->uid = pk->uid; pTQueryInfo->lastKey = pk->lastKey; pTQueryInfo->groupIndex = i; @@ -7197,36 +7304,53 @@ static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t n } STimeWindow win = {0, INT64_MAX}; - createTableQueryInfo(pInfo->pTableQueryInfo, false, win); - - return TSDB_CODE_SUCCESS; + createTableQueryInfo(pTableQueryInfo, false, win); + return pTableQueryInfo; } -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResultBlock, +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } - int32_t numOfRows = 1; //(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + int32_t numOfRows = 1; + int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResultBlock, pTaskInfo->id.str); + pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS || pInfo->pTableQueryInfo == NULL) { + goto _error; + } - initAggInfo(pInfo, pExprInfo, numOfRows, pResultBlock, pTableGroupInfo); setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_AGG; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = exprArrayDup(pExprInfo); - pOperator->numOfOutput = taosArrayGetSize(pExprInfo); + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->nextDataFn = doAggregate; + pOperator->_openFn = doOpenAggregateOptr; + pOperator->getNextFn = getAggregateResult; pOperator->closeFn = destroyAggOperatorInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } return pOperator; + _error: + destroyAggOperatorInfo(pInfo, numOfCols); + tfree(pInfo); + tfree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; } static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { @@ -7239,33 +7363,41 @@ static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { +void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { SOptrBasicInfo* pInfo = (SOptrBasicInfo*) param; doDestroyBasicInfo(pInfo, numOfOutput); } -static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { + +void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); tfree(pInfo->prevData); } -static void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { + +void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { - SSWindowOperatorInfo* pInfo = (SSWindowOperatorInfo*) param; +void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + cleanupAggSup(&pInfo->aggSup); } -static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { +void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { + SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); tfree(pInfo->p); } -static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { +void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); tfree(pInfo->prevData); @@ -7311,12 +7443,26 @@ static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput) } } -SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { +void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { + SExchangeInfo* pExInfo = (SExchangeInfo*) param; + taosArrayDestroy(pExInfo->pSources); + taosArrayDestroy(pExInfo->pSourceDataInfo); + if (pExInfo->pResult != NULL) { + blockDataDestroy(pExInfo->pResult); + } + + tsem_destroy(&pExInfo->ready); +} + +SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); int32_t numOfRows = 1; - size_t numOfOutput = taosArrayGetSize(pExprInfo); - initAggInfo(pInfo, pExprInfo, numOfRows, pResBlock, pTableGroupInfo); + int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); + pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS || pInfo->pTableQueryInfo == NULL) { + goto _error; + } size_t tableGroup = taosArrayGetSize(pTableGroupInfo->pGroupList); initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup); @@ -7327,18 +7473,24 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = exprArrayDup(pExprInfo); - pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->nextDataFn = doMultiTableAggregate; + pOperator->getNextFn = doMultiTableAggregate; pOperator->closeFn = destroyAggOperatorInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } return pOperator; + +_error: + return NULL; } -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -7346,7 +7498,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExp } pInfo->binfo.pRes = pResBlock; - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, num, &pInfo->binfo.rowCellInfoOffset); if (pInfo->binfo.pCtx == NULL) { goto _error; } @@ -7359,10 +7511,10 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExp pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = exprArrayDup(pExprInfo); - pOperator->numOfOutput = taosArrayGetSize(pExprInfo); - - pOperator->nextDataFn = doProjectOperation; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = num; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doProjectOperation; pOperator->closeFn = destroyProjectOperatorInfo; pOperator->pTaskInfo = pTaskInfo; @@ -7411,58 +7563,85 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 return 0; } -SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream) { +SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfDownstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { + ASSERT(numOfDownstream == 1); SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); - pInfo->limit = pRuntimeEnv->pQueryAttr->limit.limit; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->limit = *pLimit; + pInfo->currentOffset = pLimit->offset; pOperator->name = "LimitOperator"; -// pOperator->operatorType = OP_Limit; +// pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LIMIT; pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->nextDataFn = doLimit; + pOperator->status = OP_NOT_OPENED; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doLimit; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - int32_t code = appendDownstream(pOperator, &downstream, 1); + pOperator->pTaskInfo = pTaskInfo; + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; + _error: + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; } -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, + const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - size_t numOfOutput = taosArrayGetSize(pExprInfo); - doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, numOfOutput); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } pInfo->order = TSDB_ORDER_ASC; pInfo->precision = TSDB_TIME_PRECISION_MICRO; pInfo->win = pTaskInfo->window; pInfo->interval = *pInterval; - int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); + pInfo->win.skey = INT64_MIN; + pInfo->win.ekey = INT64_MAX; - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); + int32_t numOfRows = 4096; + int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); +// pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS/* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TimeIntervalAggOperator"; - // pOperator->operatorType = OP_TimeWindow; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = exprArrayDup(pExprInfo); - + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = taosArrayGetSize(pExprInfo); + pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->nextDataFn = doIntervalAgg; - pOperator->closeFn = destroyBasicOperatorInfo; + pOperator->_openFn = doOpenIntervalAgg; + pOperator->getNextFn = doIntervalAgg; + pOperator->closeFn = destroyIntervalOperatorInfo; code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + tfree(pInfo); + tfree(pOperator); + pTaskInfo->code = code; + return NULL; } SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { @@ -7482,7 +7661,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = doAllIntervalAgg; + pOperator->getNextFn = doAllIntervalAgg; pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7506,36 +7685,53 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = doStateWindowAgg; + pOperator->getNextFn = doStateWindowAgg; pOperator->closeFn = destroyStateWindowOperatorInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } -SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); - - pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - pInfo->prevTs = INT64_MIN; - pInfo->reptScan = false; +SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { + SSessionAggOperatorInfo* pInfo = calloc(1, sizeof(SSessionAggOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + int32_t code = doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, numOfCols, pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + + pInfo->binfo.pRes = pResBlock; + pInfo->prevTs = INT64_MIN; + pInfo->reptScan = false; pOperator->name = "SessionWindowAggOperator"; // pOperator->operatorType = OP_SessionWindow; pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = doSessionWindowAgg; - pOperator->closeFn = destroySWindowOperatorInfo; + pOperator->getNextFn = doSessionWindowAgg; + pOperator->closeFn = destroySWindowOperatorInfo; + pOperator->pTaskInfo = pTaskInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; + + _error: + if (pInfo != NULL) { + destroySWindowOperatorInfo(pInfo, numOfCols); + } + + tfree(pInfo); + tfree(pOperator); + pTaskInfo->code = code; + return NULL; } SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { @@ -7555,7 +7751,7 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = doSTableIntervalAgg; + pOperator->getNextFn = doSTableIntervalAgg; pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7579,7 +7775,7 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = doAllSTableIntervalAgg; + pOperator->getNextFn = doAllSTableIntervalAgg; pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7611,7 +7807,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = hashGroupbyAggregate; + pOperator->getNextFn = hashGroupbyAggregate; pOperator->closeFn = destroyGroupbyOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7650,7 +7846,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = doFill; + pOperator->getNextFn = doFill; pOperator->closeFn = destroySFillOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7660,15 +7856,13 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); - pInfo->slimit = pQueryAttr->slimit; - pInfo->limit = pQueryAttr->limit; - pInfo->capacity = pRuntimeEnv->resultInfo.capacity; - pInfo->threshold = (int64_t)(pInfo->capacity * 0.8); - pInfo->currentOffset = pQueryAttr->limit.offset; - pInfo->currentGroupOffset = pQueryAttr->slimit.offset; +// pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); +// pInfo->slimit = pQueryAttr->slimit; +// pInfo->limit = pQueryAttr->limit; +// pInfo->capacity = pRuntimeEnv->resultInfo.capacity; +// pInfo->threshold = (int64_t)(pInfo->capacity * 0.8); +// pInfo->currentOffset = pQueryAttr->limit.offset; +// pInfo->currentGroupOffset = pQueryAttr->slimit.offset; pInfo->multigroupResult= multigroupResult; // TODO refactor @@ -7705,7 +7899,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI return pOperator; } -static SSDataBlock* doTagScan(void* param, bool* newgroup) { +static SSDataBlock* doTagScan(SOperatorInfo *pOperator, bool* newgroup) { #if 0 SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { @@ -7853,7 +8047,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->nextDataFn = doTagScan; + pOperator->getNextFn = doTagScan; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; @@ -7909,8 +8103,7 @@ static void buildMultiDistinctKey(SDistinctOperatorInfo *pInfo, SSDataBlock *pBl } } -static SSDataBlock* hashDistinct(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* hashDistinct(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -7923,7 +8116,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); + pBlock = pOperator->pDownstream[0]->getNextFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -7995,7 +8188,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->nextDataFn = hashDistinct; + pOperator->getNextFn = hashDistinct; pOperator->pExpr = pExpr; pOperator->closeFn = destroyDistinctOperatorInfo; @@ -8082,12 +8275,14 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i return s; } -SArray* createExprInfo(SNodeList* pNodeList) { +SExprInfo* createExprInfo(SNodeList* pNodeList, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); - SArray* pArray = taosArrayInit(numOfFuncs, POINTER_BYTES); + *numOfExprs = numOfFuncs; + SExprInfo* pExprs = calloc(numOfFuncs, sizeof(SExprInfo)); + for(int32_t i = 0; i < numOfFuncs; ++i) { - SExprInfo* pExp = calloc(1, sizeof(SExprInfo)); + SExprInfo* pExp = &pExprs[i]; pExp->pExpr = calloc(1, sizeof(tExprNode)); pExp->pExpr->_function.num = 1; @@ -8137,11 +8332,9 @@ SArray* createExprInfo(SNodeList* pNodeList) { pCol->dataBlockId = pcn->dataBlockId; } } - - taosArrayPush(pArray, &pExp); } - return pArray; + return pExprs; } static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { @@ -8216,12 +8409,15 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa size_t size = LIST_LENGTH(pPhyNode->pChildren); assert(size == 1); - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); - SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + for (int32_t i = 0; i < size; ++i) { + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); - SArray* pExprInfo = createExprInfo(((SProjectPhysiNode*)pPhyNode)->pProjections); - SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - return createProjectOperatorInfo(op, pExprInfo, pResBlock, pTaskInfo); + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(((SProjectPhysiNode*)pPhyNode)->pProjections, &num); + SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); + return createProjectOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo); + } } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == nodeType(pPhyNode)) { size_t size = LIST_LENGTH(pPhyNode->pChildren); assert(size == 1); @@ -8230,9 +8426,27 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); - SArray* pExprInfo = createExprInfo(((SAggPhysiNode*)pPhyNode)->pAggFuncs); + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(((SAggPhysiNode*)pPhyNode)->pAggFuncs, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - return createAggregateOperatorInfo(op, pExprInfo, pResBlock, pTaskInfo, pTableGroupInfo); + return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); + } + } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == nodeType(pPhyNode)) { + size_t size = LIST_LENGTH(pPhyNode->pChildren); + assert(size == 1); + + for (int32_t i = 0; i < size; ++i) { + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + + SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->pFuncs, &num); + SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); + + SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = 'a', .slidingUnit = 'a'}; + return createIntervalOperatorInfo(op, pExprInfo, num, pResBlock, &interval, pTableGroupInfo, pTaskInfo); } } /*else if (pPhyNode->info.type == OP_MultiTableAggregate) { size_t size = taosArrayGetSize(pPhyNode->pChildren); diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 9f3d694c42b08a8c8222adce1b71c3c3151d2c15..28319469ccc0ad039490de31cea5d55a6350198e 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -413,7 +413,7 @@ int32_t tHashRemove(SLHashObj* pHashObj, const void *key, size_t keyLen) { void tHashPrint(const SLHashObj* pHashObj, int32_t type) { printf("==================== linear hash ====================\n"); - printf("total bucket:%d, size:%ld, ratio:%.2f\n", pHashObj->numOfBuckets, pHashObj->size, LHASH_CAP_RATIO); + printf("total bucket:%d, size:%" PRId64 ", ratio:%.2f\n", pHashObj->numOfBuckets, pHashObj->size, LHASH_CAP_RATIO); dBufSetPrintInfo(pHashObj->pBuf); @@ -425,4 +425,4 @@ void tHashPrint(const SLHashObj* pHashObj, int32_t type) { } else { dBufPrintStatis(pHashObj->pBuf); } -} \ No newline at end of file +} diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 34dd248ba7056dea986cc92aa4fb753026dd4646..08bab762be048b5a6db1a1205a00853e667e1eb8 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -37,7 +37,6 @@ typedef struct SSortHandle { SArray *pOrderInfo; bool nullFirst; - bool hasVarCol; SArray *pOrderedSource; _sort_fetch_block_fn_t fetchfp; @@ -77,6 +76,10 @@ static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { colInfo.info.bytes = pSchema[i].bytes; colInfo.info.colId = pSchema[i].colId; taosArrayPush(pBlock->pDataBlock, &colInfo); + + if (IS_VAR_DATA_TYPE(colInfo.info.type)) { + pBlock->info.hasVarCol = true; + } } return pBlock; @@ -155,7 +158,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { while(start < pDataBlock->info.rows) { int32_t stop = 0; - blockDataSplitRows(pDataBlock, pHandle->hasVarCol, start, &stop, pHandle->pageSize); + blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pHandle->pageSize); SSDataBlock* p = blockDataExtractBlock(pDataBlock, start, stop - start + 1); if (p == NULL) { return terrno; @@ -179,7 +182,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { start = stop + 1; } - blockDataClearup(pDataBlock, pHandle->hasVarCol); + blockDataClearup(pDataBlock); SSDataBlock* pBlock = createOneDataBlock(pDataBlock); int32_t code = doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId); @@ -309,7 +312,7 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa } static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity) { - blockDataClearup(pHandle->pDataBlock, pHandle->hasVarCol); + blockDataClearup(pHandle->pDataBlock); while(1) { if (cmpParam->numOfSources == pHandle->numOfCompletedSources) { @@ -475,7 +478,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); - blockDataClearup(pDataBlock, pHandle->hasVarCol); + blockDataClearup(pDataBlock); } tMergeTreeDestroy(pHandle->pMergeTree); diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index bf581c81a06f09d3940243121defba2abff754d9..36ed879e4a5f9334cba5f5857ece44a80162c0ac 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -55,8 +55,7 @@ typedef struct SDummyInputInfo { SSDataBlock* pBlock; } SDummyInputInfo; -SSDataBlock* getDummyBlock(void* param, bool* newgroup) { - SOperatorInfo* pOperator = static_cast(param); +SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { SDummyInputInfo* pInfo = static_cast(pOperator->info); if (pInfo->current >= pInfo->totalPages) { return NULL; @@ -87,7 +86,7 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { // // taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); } else { - blockDataClearup(pInfo->pBlock, true); + blockDataClearup(pInfo->pBlock); } SSDataBlock* pBlock = pInfo->pBlock; @@ -122,8 +121,7 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { return pBlock; } -SSDataBlock* get2ColsDummyBlock(void* param, bool* newgroup) { - SOperatorInfo* pOperator = static_cast(param); +SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { SDummyInputInfo* pInfo = static_cast(pOperator->info); if (pInfo->current >= pInfo->totalPages) { return NULL; @@ -153,7 +151,7 @@ SSDataBlock* get2ColsDummyBlock(void* param, bool* newgroup) { taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); } else { - blockDataClearup(pInfo->pBlock, false); + blockDataClearup(pInfo->pBlock); } SSDataBlock* pBlock = pInfo->pBlock; @@ -201,9 +199,9 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_ pOperator->name = "dummyInputOpertor4Test"; if (numOfCols == 1) { - pOperator->nextDataFn = getDummyBlock; + pOperator->getNextFn = getDummyBlock; } else { - pOperator->nextDataFn = get2ColsDummyBlock; + pOperator->getNextFn = get2ColsDummyBlock; } SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index d71ff789ba8342ed231a20e1ba8b686d93e959c0..906d4f63fb72470a5f4c928ad2fac64263b673bf 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -46,13 +46,6 @@ extern SAggFunctionInfo aggFunc[35]; #define DATA_SET_FLAG ',' // to denote the output area has data, not null value #define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) -#define TOP_BOTTOM_QUERY_LIMIT 100 - -#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) -#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY)) -#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) -#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) - typedef struct SInterpInfoDetail { TSKEY ts; // interp specified timestamp int8_t type; @@ -61,9 +54,6 @@ typedef struct SInterpInfoDetail { #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowEntryInfo))) -#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) -#define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) - typedef struct STwaInfo { int8_t hasResult; // flag to denote has value double dOutput; @@ -71,8 +61,6 @@ typedef struct STwaInfo { STimeWindow win; } STwaInfo; -extern int32_t functionCompatList[]; // compatible check array list - bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** diff --git a/source/libs/function/inc/tscript.h b/source/libs/function/inc/tscript.h index 281fe6f67988a10df36b0124d4a613587de4ec57..823e2d61f2baa9a107c59608d757de0f673cc38b 100644 --- a/source/libs/function/inc/tscript.h +++ b/source/libs/function/inc/tscript.h @@ -70,7 +70,7 @@ typedef struct { SList *scriptEnvs; // int32_t mSize; // pool limit int32_t cSize; // current available size - pthread_mutex_t mutex; + TdThreadMutex mutex; } ScriptEnvPool; ScriptCtx* createScriptCtx(char *str, int8_t resType, int16_t resBytes); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index ccac37fd0c903885d330020bd49ba3ca551d6e82..fa83068755c511ad57cce787a3d1361a10a6d61c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -52,10 +52,6 @@ static void doFinalizer(SResultRowEntryInfo* pResInfo) { cleanupResultRowEntry(p void functionFinalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult != DATA_SET_FLAG) { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } - doFinalizer(pResInfo); } @@ -396,7 +392,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int32_t *pData = (int32_t*)pCol->pData; int32_t *val = (int32_t*) buf; - for (int32_t i = 0; i < pCtx->size; ++i) { + for (int32_t i = start; i < start + numOfRows; ++i) { if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) { continue; } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 41b0126a073ef0e22c253406af592d9f1f46621d..f9f2e90770bced6845bd86e4db417e11ec1038ee 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -26,7 +26,7 @@ typedef struct SFuncMgtService { } SFuncMgtService; static SFuncMgtService gFunMgtService; -static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT; +static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; static int32_t initFunctionCode = 0; static void doInitFunctionHashTable() { @@ -45,7 +45,7 @@ static void doInitFunctionHashTable() { } int32_t fmFuncMgtInit() { - pthread_once(&functionHashTableInit, doInitFunctionHashTable); + taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable); return initFunctionCode; } diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c deleted file mode 100644 index e302643c328415553918bfad0ad05e72b9d20ebb..0000000000000000000000000000000000000000 --- a/source/libs/function/src/tfunction.c +++ /dev/null @@ -1,415 +0,0 @@ -#include "os.h" -#include "tarray.h" -#include "function.h" -#include "thash.h" -#include "taggfunction.h" - -static SHashObj* functionHashTable = NULL; -static SHashObj* udfHashTable = NULL; - -static void doInitFunctionHashTable() { - int numOfEntries = tListLen(aggFunc); - functionHashTable = taosHashInit(numOfEntries, MurmurHash3_32, false, false); - for (int32_t i = 0; i < numOfEntries; i++) { - int32_t len = (uint32_t)strlen(aggFunc[i].name); - - SAggFunctionInfo* ptr = &aggFunc[i]; - taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&ptr, POINTER_BYTES); - } - -/* - numOfEntries = tListLen(scalarFunc); - for(int32_t i = 0; i < numOfEntries; ++i) { - int32_t len = (int32_t) strlen(scalarFunc[i].name); - SScalarFunctionInfo* ptr = &scalarFunc[i]; - taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES); - } -*/ - - udfHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, true); -} - -static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT; - -int32_t qIsBuiltinFunction(const char* name, int32_t len, bool* scalarFunction) { - pthread_once(&functionHashTableInit, doInitFunctionHashTable); - - SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, name, len); - if (pInfo != NULL) { - *scalarFunction = ((*pInfo)->type == FUNCTION_TYPE_SCALAR); - return (*pInfo)->functionId; - } else { - return -1; - } -} - -bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId) { - return true; -} - -bool qIsAggregateFunction(const char* functionName) { - assert(functionName != NULL); - bool scalarfunc = false; - qIsBuiltinFunction(functionName, strlen(functionName), &scalarfunc); - - return !scalarfunc; -} - -bool qIsSelectivityFunction(const char* functionName) { - assert(functionName != NULL); - pthread_once(&functionHashTableInit, doInitFunctionHashTable); - - size_t len = strlen(functionName); - SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, functionName, len); - if (pInfo != NULL) { - return ((*pInfo)->status | FUNCSTATE_SELECTIVITY) != 0; - } - - return false; -} - -SAggFunctionInfo* qGetFunctionInfo(const char* name, int32_t len) { - pthread_once(&functionHashTableInit, doInitFunctionHashTable); - - SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, name, len); - if (pInfo != NULL) { - return (*pInfo); - } else { - return NULL; - } -} - -void qAddUdfInfo(uint64_t id, SUdfInfo* pUdfInfo) { - int32_t len = (uint32_t)strlen(pUdfInfo->name); - taosHashPut(udfHashTable, pUdfInfo->name, len, (void*)&pUdfInfo, POINTER_BYTES); -} - -void qRemoveUdfInfo(uint64_t id, SUdfInfo* pUdfInfo) { - int32_t len = (uint32_t)strlen(pUdfInfo->name); - taosHashRemove(udfHashTable, pUdfInfo->name, len); -} - -bool isTagsQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - char* f = *(char**) taosArrayGet(pFunctionIdList, i); - - // todo handle count(tbname) query - if (strcmp(f, "project") != 0 && strcmp(f, "count") != 0) { - return false; - } - - // "select count(tbname)" query -// if (functId == FUNCTION_COUNT && pExpr->base.colpDesc->colId == TSDB_TBNAME_COLUMN_INDEX) { -// continue; -// } - } - - return true; -} - -//bool tscMultiRoundQuery(SArray* pFunctionIdList, int32_t index) { -// if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) { -// return false; -// } -// -// size_t numOfExprs = (int32_t) getNumOfExprs(pQueryInfo); -// for(int32_t i = 0; i < numOfExprs; ++i) { -// SExprInfo* pExpr = getExprInfo(pQueryInfo, i); -// if (pExpr->base.functionId == FUNCTION_STDDEV_DST) { -// return true; -// } -// } -// -// return false; -//} - -bool isProjectionQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - char* f = *(char**) taosArrayGet(pFunctionIdList, i); - if (strcmp(f, "project") == 0) { - return true; - } - } - - return false; -} - -bool isDiffDerivativeQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); - if (f == FUNCTION_TS_DUMMY) { - continue; - } - - if (f == FUNCTION_DIFF || f == FUNCTION_DERIVATIVE) { - return true; - } - } - - return false; -} - -bool isInterpQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); - if (f == FUNCTION_TAG || f == FUNCTION_TS) { - continue; - } - - if (f != FUNCTION_INTERP) { - return false; - } - } - - return true; -} - -bool isArithmeticQueryOnAggResult(SArray* pFunctionIdList) { - if (isProjectionQuery(pFunctionIdList)) { - return false; - } - - assert(0); - -// size_t numOfOutput = getNumOfFields(pQueryInfo); -// for(int32_t i = 0; i < numOfOutput; ++i) { -// SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pExpr; -// if (pExprInfo->pExpr != NULL) { -// return true; -// } -// } - - return false; -} - -bool isGroupbyColumn(SGroupbyExpr* pGroupby) { - return !pGroupby->groupbyTag; -} - -bool isTopBotQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - char* f = *(char**) taosArrayGet(pFunctionIdList, i); - if (strcmp(f, "project") == 0) { - continue; - } - - if (strcmp(f, "top") == 0 || strcmp(f, "bottom") == 0) { - return true; - } - } - - return false; -} - -bool isTsCompQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - if (num != 1) { - return false; - } - - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, 0); - return f == FUNCTION_TS_COMP; -} - -bool isTWAQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); - if (f == FUNCTION_TWA) { - return true; - } - } - - return false; -} - -bool isIrateQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); - if (f == FUNCTION_IRATE) { - return true; - } - } - - return false; -} - -bool isStabledev(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); - if (f == FUNCTION_STDDEV_DST) { - return true; - } - } - - return false; -} - -bool needReverseScan(SArray* pFunctionIdList) { - assert(0); - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < num; ++i) { - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); - if (f == FUNCTION_TS || f == FUNCTION_TS_DUMMY || f == FUNCTION_TAG) { - continue; - } - -// if ((f == FUNCTION_FIRST || f == FUNCTION_FIRST_DST) && pQueryInfo->order.order == TSDB_ORDER_DESC) { -// return true; -// } - - if (f == FUNCTION_LAST || f == FUNCTION_LAST_DST) { - // the scan order to acquire the last result of the specified column -// int32_t order = (int32_t)pExpr->base.param[0].i64; -// if (order != pQueryInfo->order.order) { -// return true; -// } - } - } - - return false; -} - -bool isAgg(SArray* pFunctionIdList) { - size_t size = taosArrayGetSize(pFunctionIdList); - for (int32_t i = 0; i < size; ++i) { - char* f = *(char**) taosArrayGet(pFunctionIdList, i); - if (strcmp(f, "project") == 0) { - return false; - } - - if (qIsAggregateFunction(f)) { - return true; - } - } - - return false; -} - -bool isBlockDistQuery(SArray* pFunctionIdList) { - int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - char* f = *(char**) taosArrayGet(pFunctionIdList, 0); - return (num == 1 && strcmp(f, "block_dist") == 0); -} - -bool isTwoStageSTableQuery(SArray* pFunctionIdList, int32_t tableIndex) { -// if (pQueryInfo == NULL) { -// return false; -// } -// -// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); -// if (pTableMetaInfo == NULL) { -// return false; -// } -// -// if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { -// return false; -// } -// -// // for ordered projection query, iterate all qualified vnodes sequentially -// if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, tableIndex)) { -// return false; -// } -// -// if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) { -// return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); -// } - - return false; -} - -bool isProjectionQueryOnSTable(SArray* pFunctionIdList, int32_t tableIndex) { -// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); -// -// /* -// * In following cases, return false for non ordered project query on super table -// * 1. failed to get tableMeta from server; 2. not a super table; 3. limitation is 0; -// * 4. show queries, instead of a select query -// */ -// size_t numOfExprs = getNumOfExprs(pQueryInfo); -// if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || -// pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) { -// return false; -// } -// -// for (int32_t i = 0; i < numOfExprs; ++i) { -// int32_t functionId = getExprInfo(pQueryInfo, i)->base.functionId; -// -// if (functionId < 0) { -// SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); -// if (pUdfInfo->funcType == TSDB_FUNC_TYPE_AGGREGATE) { -// return false; -// } -// -// continue; -// } -// -// if (functionId != FUNCTION_PRJ && -// functionId != FUNCTION_TAGPRJ && -// functionId != FUNCTION_TAG && -// functionId != FUNCTION_TS && -// functionId != FUNCTION_ARITHM && -// functionId != FUNCTION_TS_COMP && -// functionId != FUNCTION_DIFF && -// functionId != FUNCTION_DERIVATIVE && -// functionId != FUNCTION_TS_DUMMY && -// functionId != FUNCTION_TID_TAG) { -// return false; -// } -// } - - return true; -} - -bool hasTagValOutput(SArray* pFunctionIdList) { - size_t size = taosArrayGetSize(pFunctionIdList); - - // if (numOfExprs == 1 && pExpr1->base.functionId == FUNCTION_TS_COMP) { -// return true; -// } - - for (int32_t i = 0; i < size; ++i) { - int32_t functionId = *(int16_t*) taosArrayGet(pFunctionIdList, i); - - // ts_comp column required the tag value for join filter - if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { - return true; - } - } - - return false; -} - -//bool timeWindowInterpoRequired(SArray* pFunctionIdList) { -// int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); -// for (int32_t i = 0; i < num; ++i) { -// int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); -// if (f == FUNCTION_TWA || f == FUNCTION_INTERP) { -// return true; -// } -// } -// -// return false; -//} - -void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc) { - assert(pFunctionIdList != NULL); - - pDesc->blockDistribution = isBlockDistQuery(pFunctionIdList); - if (pDesc->blockDistribution) { - return; - } - -// pDesc->projectionQuery = isProjectionQuery(pFunctionIdList); -// pDesc->onlyTagQuery = isTagsQuery(pFunctionIdList); - pDesc->interpQuery = isInterpQuery(pFunctionIdList); - pDesc->topbotQuery = isTopBotQuery(pFunctionIdList); - pDesc->agg = isAgg(pFunctionIdList); -} diff --git a/source/libs/function/src/tscript.c b/source/libs/function/src/tscript.c index 7c07b0b7831e685c88ba861b00f27e19f094161d..25cf3cfbb16598abde4c911bdb1c27a50fe4303c 100644 --- a/source/libs/function/src/tscript.c +++ b/source/libs/function/src/tscript.c @@ -333,7 +333,7 @@ void destroyLuaEnv(lua_State *lua) { int32_t scriptEnvPoolInit() { const int size = 10; // configure or not pool = malloc(sizeof(ScriptEnvPool)); - pthread_mutex_init(&pool->mutex, NULL); + taosThreadMutexInit(&pool->mutex, NULL); pool->scriptEnvs = tdListNew(sizeof(ScriptEnv *)); for (int i = 0; i < size; i++) { @@ -359,7 +359,7 @@ void scriptEnvPoolCleanup() { listNodeFree(pNode); } tdListFree(pool->scriptEnvs); - pthread_mutex_destroy(&pool->mutex); + taosThreadMutexDestroy(&pool->mutex); free(pool); } @@ -372,9 +372,9 @@ void destroyScriptEnv(ScriptEnv *pEnv) { ScriptEnv* getScriptEnvFromPool() { ScriptEnv *pEnv = NULL; - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); if (pool->cSize <= 0) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return NULL; } SListNode *pNode = tdListPopHead(pool->scriptEnvs); @@ -384,7 +384,7 @@ ScriptEnv* getScriptEnvFromPool() { } pool->cSize--; - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return pEnv; } @@ -392,11 +392,11 @@ void addScriptEnvToPool(ScriptEnv *pEnv) { if (pEnv == NULL) { return; } - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); lua_settop(pEnv->lua_state, 0); tdListAppend(pool->scriptEnvs, (void *)(&pEnv)); pool->cSize++; - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); } bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len) { diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 3d1d5356c24e23a3e89eedd59ca2a58b1e4d01ae..37318767c71e8558d284cccb05c6c5d77f48cbdf 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -58,7 +58,7 @@ struct SIndex { char* path; SIndexStat stat; - pthread_mutex_t mtx; + TdThreadMutex mtx; }; struct SIndexOpts { diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index a6ebcd6d6ffbf700c482e8094754ebe1c7e54742..086e75d99fd85887701daad11b88310c417c9bfe 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -43,8 +43,8 @@ typedef struct IndexCache { int8_t type; uint64_t suid; - pthread_mutex_t mtx; - pthread_cond_t finished; + TdThreadMutex mtx; + TdThreadCond finished; } IndexCache; #define CACHE_VERSION(cache) atomic_load_32(&cache->version) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 5da0dc537bfbefbe79a430d0220cef3149eb7f7b..cf5c3f306b5e56e15221ede6436ce0340be80ccd 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -260,7 +260,7 @@ typedef struct Fst { FstMeta* meta; FstSlice* data; // FstNode* root; // - pthread_mutex_t mtx; + TdThreadMutex mtx; } Fst; // refactor simple function diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 744f6ca70bb5dbbb2b7233e188044ae626da61cd..83410306eb7b3fdc4c56b1283219a8c6122acd37 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -45,7 +45,7 @@ typedef struct SIdxColInfo { int cVersion; } SIdxColInfo; -static pthread_once_t isInit = PTHREAD_ONCE_INIT; +static TdThreadOnce isInit = PTHREAD_ONCE_INIT; // static void indexInit(); static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); @@ -61,7 +61,7 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, Iterat // int32_t indexSerialKey(ICacheKey* key, char* buf); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { - pthread_once(&isInit, indexInit); + taosThreadOnce(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); if (sIdx == NULL) { return -1; @@ -82,7 +82,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; sIdx->path = tstrdup(path); - pthread_mutex_init(&sIdx->mtx, NULL); + taosThreadMutexInit(&sIdx->mtx, NULL); *index = sIdx; return 0; #endif @@ -112,7 +112,7 @@ void indexClose(SIndex* sIdx) { iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); - pthread_mutex_destroy(&sIdx->mtx); + taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); #endif free(sIdx->path); @@ -140,7 +140,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { #ifdef USE_INVERTED_INDEX // TODO(yihao): reduce the lock range - pthread_mutex_lock(&index->mtx); + taosThreadMutexLock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -154,7 +154,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } - pthread_mutex_unlock(&index->mtx); + taosThreadMutexUnlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -240,6 +240,7 @@ int indexRebuild(SIndex* index, SIndexOpts* opts){ #ifdef USE_INVERTED_INDEX #endif + return 0; } SIndexOpts* indexOptsCreate() { @@ -332,10 +333,10 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; int32_t sz = indexSerialCacheKey(&key, buf); - pthread_mutex_lock(&sIdx->mtx); + taosThreadMutexLock(&sIdx->mtx); IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); cache = (pCache == NULL) ? NULL : *pCache; - pthread_mutex_unlock(&sIdx->mtx); + taosThreadMutexUnlock(&sIdx->mtx); *result = taosArrayInit(4, sizeof(uint64_t)); // TODO: iterator mem and tidex @@ -563,10 +564,10 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { TFileHeader* header = &reader->header; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; - pthread_mutex_lock(&sIdx->mtx); + taosThreadMutexLock(&sIdx->mtx); IndexTFile* ifile = (IndexTFile*)sIdx->tindex; tfileCachePut(ifile->cache, &key, reader); - pthread_mutex_unlock(&sIdx->mtx); + taosThreadMutexUnlock(&sIdx->mtx); return ret; END: if (tw != NULL) { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index b40ded9e9ace2763950a93f272cb309074febc2c..2b4327b091897c12293c582041a7a5fbc4fbfeb5 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -54,8 +54,8 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in cache->suid = suid; cache->occupiedMem = 0; - pthread_mutex_init(&cache->mtx, NULL); - pthread_cond_init(&cache->finished, NULL); + taosThreadMutexInit(&cache->mtx, NULL); + taosThreadCondInit(&cache->finished, NULL); indexCacheRef(cache); return cache; @@ -63,10 +63,10 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in void indexCacheDebug(IndexCache* cache) { MemTable* tbl = NULL; - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->mem; indexMemRef(tbl); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); { SSkipList* slt = tbl->mem; @@ -85,10 +85,10 @@ void indexCacheDebug(IndexCache* cache) { } { - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->imm; indexMemRef(tbl); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); if (tbl != NULL) { SSkipList* slt = tbl->mem; SSkipListIterator* iter = tSkipListCreateIter(slt); @@ -126,13 +126,13 @@ void indexCacheDestroyImm(IndexCache* cache) { } MemTable* tbl = NULL; - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread - pthread_cond_broadcast(&cache->finished); + taosThreadCondBroadcast(&cache->finished); - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); indexMemUnRef(tbl); indexMemUnRef(tbl); @@ -146,8 +146,8 @@ void indexCacheDestroy(void* cache) { indexMemUnRef(pCache->imm); free(pCache->colName); - pthread_mutex_destroy(&pCache->mtx); - pthread_cond_destroy(&pCache->finished); + taosThreadMutexDestroy(&pCache->mtx); + taosThreadCondDestroy(&pCache->finished); free(pCache); } @@ -158,7 +158,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { return NULL; } - pthread_mutex_lock(&cache->mtx); + taosThreadMutexLock(&cache->mtx); indexMemRef(cache->imm); @@ -169,7 +169,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { iiter->next = indexCacheIteratorNext; iiter->getValue = indexCacheIteratorGetValue; - pthread_mutex_unlock(&cache->mtx); + taosThreadMutexUnlock(&cache->mtx); return iiter; } @@ -190,14 +190,17 @@ int indexCacheSchedToMerge(IndexCache* pCache) { schedMsg.msg = NULL; taosScheduleTask(indexQhandle, &schedMsg); + + return 0; } + static void indexCacheMakeRoomForWrite(IndexCache* cache) { while (true) { if (cache->occupiedMem * MEM_ESTIMATE_RADIO < MEM_THRESHOLD) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - pthread_cond_wait(&cache->finished, &cache->mtx); + taosThreadCondWait(&cache->finished, &cache->mtx); } else { indexCacheRef(cache); cache->imm = cache->mem; @@ -237,7 +240,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { // ugly code, refactor later int64_t estimate = sizeof(ct) + strlen(ct->colVal); - pthread_mutex_lock(&pCache->mtx); + taosThreadMutexLock(&pCache->mtx); pCache->occupiedMem += estimate; indexCacheMakeRoomForWrite(pCache); MemTable* tbl = pCache->mem; @@ -245,7 +248,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { tSkipListPut(tbl->mem, (char*)ct); indexMemUnRef(tbl); - pthread_mutex_unlock(&pCache->mtx); + taosThreadMutexUnlock(&pCache->mtx); indexCacheUnRef(pCache); return 0; @@ -296,12 +299,12 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; - pthread_mutex_lock(&pCache->mtx); + taosThreadMutexLock(&pCache->mtx); mem = pCache->mem; imm = pCache->imm; indexMemRef(mem); indexMemRef(imm); - pthread_mutex_unlock(&pCache->mtx); + taosThreadMutexUnlock(&pCache->mtx); SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index a6cabbd43940c91fb62afe9c65fc9dc4bb4139c3..58d5a871cd4e481c341d52c39cd3a76290d6bb29 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -571,6 +571,8 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { } fstSliceDestroy(&t); } + + return 0; } // fst node function @@ -1021,26 +1023,28 @@ Fst* fstCreate(FstSlice* slice) { *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); fst->data = s; - pthread_mutex_init(&fst->mtx, NULL); + taosThreadMutexInit(&fst->mtx, NULL); return fst; FST_CREAT_FAILED: free(fst->meta); free(fst); + + return NULL; } void fstDestroy(Fst* fst) { if (fst) { free(fst->meta); fstSliceDestroy(fst->data); free(fst->data); - pthread_mutex_destroy(&fst->mtx); + taosThreadMutexDestroy(&fst->mtx); } free(fst); } bool fstGet(Fst* fst, FstSlice* b, Output* out) { // dec lock range - // pthread_mutex_lock(&fst->mtx); + // taosThreadMutexLock(&fst->mtx); FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; @@ -1053,7 +1057,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { uint8_t inp = data[i]; Output res = 0; if (false == fstNodeFindInput(root, inp, &res)) { - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); return false; } @@ -1064,7 +1068,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); @@ -1076,7 +1080,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { } taosArrayDestroy(nodes); // fst->root = NULL; - // pthread_mutex_unlock(&fst->mtx); + // taosThreadMutexUnlock(&fst->mtx); *out = tOut; return true; } @@ -1286,6 +1290,8 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { } return false; } + + return false; } StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) { diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index 6ef901410bff1923ee425ffcaff6c44eb7d047d8..452c38f66b58d13425156d4986d71d6b54d5cc63 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -48,7 +48,7 @@ typedef struct SMonInfo { } SMonInfo; typedef struct { - pthread_mutex_t lock; + TdThreadMutex lock; SArray *logs; // array of SMonLogItem int32_t maxLogs; const char *server; diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index 352b59e9316d755d51619e7499d73e53441536e3..aa5eafd8b23de6167ad1980511a66f2e3f884d81 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -23,7 +23,7 @@ static SMonitor tsMonitor = {0}; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { - pthread_mutex_lock(&tsMonitor.lock); + taosThreadMutexLock(&tsMonitor.lock); int32_t size = taosArrayGetSize(tsMonitor.logs); if (size < tsMonitor.maxLogs) { SMonLogItem item = {.ts = ts, .level = level}; @@ -32,7 +32,7 @@ void monRecordLog(int64_t ts, ELogLevel level, const char *content) { tstrncpy(pItem->content, content, MON_LOG_LEN); } } - pthread_mutex_unlock(&tsMonitor.lock); + taosThreadMutexUnlock(&tsMonitor.lock); } int32_t monInit(const SMonCfg *pCfg) { @@ -48,7 +48,7 @@ int32_t monInit(const SMonCfg *pCfg) { tsMonitor.comp = pCfg->comp; tsLogFp = monRecordLog; tsMonitor.state.time = taosGetTimestampMs(); - pthread_mutex_init(&tsMonitor.lock, NULL); + taosThreadMutexInit(&tsMonitor.lock, NULL); return 0; } @@ -56,7 +56,7 @@ void monCleanup() { tsLogFp = NULL; taosArrayDestroy(tsMonitor.logs); tsMonitor.logs = NULL; - pthread_mutex_destroy(&tsMonitor.lock); + taosThreadMutexDestroy(&tsMonitor.lock); } SMonInfo *monCreateMonitorInfo() { @@ -66,10 +66,10 @@ SMonInfo *monCreateMonitorInfo() { return NULL; } - pthread_mutex_lock(&tsMonitor.lock); + taosThreadMutexLock(&tsMonitor.lock); pMonitor->logs = taosArrayDup(tsMonitor.logs); taosArrayClear(tsMonitor.logs); - pthread_mutex_unlock(&tsMonitor.lock); + taosThreadMutexUnlock(&tsMonitor.lock); pMonitor->pJson = tjsonCreateObject(); if (pMonitor->pJson == NULL || pMonitor->logs == NULL) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index bd4fb4737c3bd8a64f642e2cb4ef62c8e8f22318..2fa7a8d663756f10b67e319ecb6640d65e3050e6 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -202,7 +202,6 @@ static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { } static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { - COPY_SCALAR_FIELD(id); CLONE_NODE_LIST_FIELD(pTargets); CLONE_NODE_FIELD(pConditions); CLONE_NODE_LIST_FIELD(pChildren); @@ -210,7 +209,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { } static STableMeta* tableMetaClone(const STableMeta* pSrc) { - int32_t len = sizeof(STableMeta) + (pSrc->tableInfo.numOfTags + pSrc->tableInfo.numOfColumns) * sizeof(SSchema); + int32_t len = TABLE_META_SIZE(pSrc); STableMeta* pDst = malloc(len); if (NULL == pDst) { return NULL; @@ -220,7 +219,7 @@ static STableMeta* tableMetaClone(const STableMeta* pSrc) { } static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) { - int32_t len = sizeof(SVgroupsInfo) + pSrc->numOfVgroups * sizeof(SVgroupInfo); + int32_t len = VGROUPS_INFO_SIZE(pSrc); SVgroupsInfo* pDst = malloc(len); if (NULL == pDst) { return NULL; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 3ede72eebd0086ce2d1a285eee4bd65313580552..f43557c33527d4d03ea45387c5d65962eee36b62 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "cmdnodes.h" #include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" @@ -85,6 +86,8 @@ const char* nodesNodeName(ENodeType type) { return "ShowDatabaseStmt"; case QUERY_NODE_SHOW_TABLES_STMT: return "ShowTablesStmt"; + case QUERY_NODE_CREATE_TOPIC_STMT: + return "CreateTopicStmt"; case QUERY_NODE_LOGIC_PLAN_SCAN: return "LogicScan"; case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -153,8 +156,7 @@ static int32_t nodeListToJson(SJson* pJson, const char* pName, const SNodeList* return TSDB_CODE_SUCCESS; } -static int32_t jsonToNodeList(const SJson* pJson, const char* pName, SNodeList** pList) { - const SJson* pJsonArray = tjsonGetObjectItem(pJson, pName); +static int32_t jsonToNodeListImpl(const SJson* pJsonArray, SNodeList** pList) { int32_t size = (NULL == pJsonArray ? 0 : tjsonGetArraySize(pJsonArray)); if (size > 0) { *pList = nodesMakeList(); @@ -178,21 +180,155 @@ static int32_t jsonToNodeList(const SJson* pJson, const char* pName, SNodeList** return code; } -static const char* jkTableMetaUid = "TableMetaUid"; -static const char* jkTableMetaSuid = "TableMetaSuid"; +static int32_t jsonToNodeList(const SJson* pJson, const char* pName, SNodeList** pList) { + return jsonToNodeListImpl(tjsonGetObjectItem(pJson, pName), pList); +} + +static const char* jkTableComInfoNumOfTags = "NumOfTags"; +static const char* jkTableComInfoPrecision = "Precision"; +static const char* jkTableComInfoNumOfColumns = "NumOfColumns"; +static const char* jkTableComInfoRowSize = "RowSize"; + +static int32_t tableComInfoToJson(const void* pObj, SJson* pJson) { + const STableComInfo* pNode = (const STableComInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfTags, pNode->numOfTags); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoPrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableComInfoRowSize, pNode->rowSize); + } + + return code; +} + +static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) { + STableComInfo* pNode = (STableComInfo*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize); + } + + return code; +} + +static const char* jkSchemaType = "Type"; +static const char* jkSchemaColId = "ColId"; +static const char* jkSchemaBytes = "bytes"; +static const char* jkSchemaName = "Name"; + +static int32_t schemaToJson(const void* pObj, SJson* pJson) { + const SSchema* pNode = (const SSchema*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSchemaType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSchemaColId, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSchemaBytes, pNode->bytes); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSchemaName, pNode->name); + } + + return code; +} + +static int32_t jsonToSchema(const SJson* pJson, void* pObj) { + SSchema* pNode = (SSchema*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkSchemaType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name); + } + + return code; +} + +static const char* jkTableMetaVgId = "VgId"; +static const char* jkTableMetaTableType = "TableType"; +static const char* jkTableMetaUid = "Uid"; +static const char* jkTableMetaSuid = "Suid"; +static const char* jkTableMetaSversion = "Sversion"; +static const char* jkTableMetaTversion = "Tversion"; +static const char* jkTableMetaComInfo = "ComInfo"; +static const char* jkTableMetaColSchemas = "ColSchemas"; static int32_t tableMetaToJson(const void* pObj, SJson* pJson) { const STableMeta* pNode = (const STableMeta*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid); + int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableMetaSuid, pNode->suid); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaSversion, pNode->sversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableMetaTversion, pNode->tversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTableMetaComInfo, tableComInfoToJson, &pNode->tableInfo); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkTableMetaColSchemas, schemaToJson, pNode->schema, sizeof(SSchema), TABLE_TOTAL_COL_NUM(pNode)); + } + + return code; +} + +static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { + STableMeta* pNode = (STableMeta*)pObj; + + int32_t code = tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkTableMetaColSchemas, jsonToSchema, pNode->schema, sizeof(SSchema)); + } return code; } -static const char* jkLogicPlanId = "Id"; static const char* jkLogicPlanTargets = "Targets"; static const char* jkLogicPlanConditions = "Conditions"; static const char* jkLogicPlanChildren = "Children"; @@ -200,10 +336,7 @@ static const char* jkLogicPlanChildren = "Children"; static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { const SLogicNode* pNode = (const SLogicNode*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkLogicPlanId, pNode->id); - if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets); - } + int32_t code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions); } @@ -214,7 +347,22 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { + SLogicNode* pNode = (SLogicNode*)pObj; + + int32_t code = jsonToNodeList(pJson, jkLogicPlanTargets, &pNode->pTargets); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkLogicPlanConditions, &pNode->pConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkLogicPlanChildren, &pNode->pChildren); + } + + return code; +} + static const char* jkScanLogicPlanScanCols = "ScanCols"; +static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize"; static const char* jkScanLogicPlanTableMeta = "TableMeta"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { @@ -224,6 +372,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta); } @@ -231,6 +382,24 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { + SScanLogicNode* pNode = (SScanLogicNode*)pObj; + + int32_t objSize = 0; + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanLogicPlanScanCols, &pNode->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize); + } + + return code; +} + static const char* jkProjectLogicPlanProjections = "Projections"; static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { @@ -244,6 +413,17 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { + SProjectLogicNode* pNode = (SProjectLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkProjectLogicPlanProjections, &pNode->pProjections); + } + + return code; +} + static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; @@ -444,6 +624,14 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { return code; } +static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { + return physiScanNodeToJson(pObj, pJson); +} + +static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { + return jsonToPhysiScanNode(pJson, pObj); +} + static const char* jkEndPointFqdn = "Fqdn"; static const char* jkEndPointPort = "Port"; @@ -663,6 +851,8 @@ static const char* jkIntervalPhysiPlanFuncs = "Funcs"; static const char* jkIntervalPhysiPlanInterval = "Interval"; static const char* jkIntervalPhysiPlanOffset = "Offset"; static const char* jkIntervalPhysiPlanSliding = "Sliding"; +static const char* jkIntervalPhysiPlanIntervalUnit = "intervalUnit"; +static const char* jkIntervalPhysiPlanSlidingUnit = "slidingUnit"; static const char* jkIntervalPhysiPlanFill = "Fill"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { @@ -684,6 +874,12 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanSliding, pNode->sliding); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanIntervalUnit, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanSlidingUnit, pNode->slidingUnit); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkIntervalPhysiPlanFill, nodeToJson, pNode->pFill); } @@ -710,6 +906,12 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkIntervalPhysiPlanSliding, &pNode->sliding); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkIntervalPhysiPlanIntervalUnit, &pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkIntervalPhysiPlanSlidingUnit, &pNode->slidingUnit); + } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkIntervalPhysiPlanFill, (SNode**)&pNode->pFill); } @@ -1321,6 +1523,161 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkTableDbName = "DbName"; +static const char* jkTableTableName = "tableName"; +static const char* jkTableTableAlias = "tableAlias"; + +static int32_t tableNodeToJson(const void* pObj, SJson* pJson) { + const STableNode* pNode = (const STableNode*)pObj; + + int32_t code = exprNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableTableAlias, pNode->tableAlias); + } + + return code; +} + +static int32_t jsonToTableNode(const SJson* pJson, void* pObj) { + STableNode* pNode = (STableNode*)pObj; + + int32_t code = jsonToExprNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkTableDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkTableTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkTableTableAlias, pNode->tableAlias); + } + + return code; +} + +static const char* jkVgroupInfoVgId = "VgId"; +static const char* jkVgroupInfoHashBegin = "HashBegin"; +static const char* jkVgroupInfoHashEnd = "HashEnd"; +static const char* jkVgroupInfoEpSet = "EpSet"; +static const char* jkVgroupInfoNumOfTable = "NumOfTable"; + +static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) { + const SVgroupInfo* pNode = (const SVgroupInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable); + } + + return code; +} + +static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) { + SVgroupInfo* pNode = (SVgroupInfo*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable); + } + + return code; +} + +static const char* jkVgroupsInfoNum = "Num"; +static const char* jkVgroupsInfoVgroups = "Vgroups"; + +static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) { + const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), pNode->numOfVgroups); + } + + return code; +} + +static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) { + SVgroupsInfo* pNode = (SVgroupsInfo*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo)); + } + + return code; +} + +static const char* jkRealTableMetaSize = "MetaSize"; +static const char* jkRealTableMeta = "Meta"; +static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize"; +static const char* jkRealTableVgroupsInfo = "VgroupsInfo"; + +static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { + const SRealTableNode* pNode = (const SRealTableNode*)pObj; + + int32_t code = tableNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRealTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkRealTableMeta, tableMetaToJson, pNode->pMeta); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRealTableVgroupsInfoSize, VGROUPS_INFO_SIZE(pNode->pVgroupList)); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkRealTableVgroupsInfo, vgroupsInfoToJson, pNode->pVgroupList); + } + + return code; +} + +static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) { + SRealTableNode* pNode = (SRealTableNode*)pObj; + + int32_t objSize = 0; + int32_t code = jsonToTableNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRealTableMetaSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkRealTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRealTableVgroupsInfoSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkRealTableVgroupsInfo, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize); + } + + return code; +} + static const char* jkGroupingSetType = "GroupingSetType"; static const char* jkGroupingSetParameter = "Parameters"; @@ -1512,7 +1869,7 @@ static const char* jkSelectStmtSlimit = "Slimit"; static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { const SSelectStmt* pNode = (const SSelectStmt*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); + int32_t code = tjsonAddBoolToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkSelectStmtProjections, pNode->pProjectionList); } @@ -1547,6 +1904,83 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) { + SSelectStmt* pNode = (SSelectStmt*)pObj; + + int32_t code = tjsonGetBoolValue(pJson, jkSelectStmtDistinct, &pNode->isDistinct); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSelectStmtProjections, &pNode->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtFrom, &pNode->pFromTable); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtWhere, &pNode->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSelectStmtPartitionBy, &pNode->pPartitionByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtWindow, &pNode->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSelectStmtGroupBy, &pNode->pGroupByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtHaving, &pNode->pHaving); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSelectStmtOrderBy, &pNode->pOrderByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtLimit, &pNode->pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSelectStmtSlimit, &pNode->pSlimit); + } + + return code; +} + +static const char* jkCreateTopicStmtTopicName = "TopicName"; +static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; +static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateTopicStmtQuery = "Query"; + +static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) { + const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery); + } + + return code; +} + +static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { + SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: @@ -1560,6 +1994,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_FUNCTION: return functionNodeToJson(pObj, pJson); case QUERY_NODE_REAL_TABLE: + return realTableNodeToJson(pObj, pJson); case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_JOIN_TABLE: break; @@ -1597,6 +2032,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_TABLES_STMT: break; + case QUERY_NODE_CREATE_TOPIC_STMT: + return createTopicStmtToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -1613,9 +2050,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiTagScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: return physiTableScanNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - break; + return physiStreamScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: return physiSysTableScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: @@ -1639,6 +2075,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN: return planToJson(pObj, pJson); default: + assert(0); break; } nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); @@ -1657,7 +2094,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicConditionNode(pJson, pObj); case QUERY_NODE_FUNCTION: return jsonToFunctionNode(pJson, pObj); - // case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_REAL_TABLE: + return jsonToRealTableNode(pJson, pObj); // case QUERY_NODE_TEMP_TABLE: // case QUERY_NODE_JOIN_TABLE: // break; @@ -1683,22 +2121,26 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDownstreamSourceNode(pJson, pObj); // case QUERY_NODE_SET_OPERATOR: // break; - // case QUERY_NODE_SELECT_STMT: - // return jsonToSelectStmt(pJson, pObj); - // case QUERY_NODE_LOGIC_PLAN_SCAN: - // return jsonToLogicScanNode(pJson, pObj); + case QUERY_NODE_SELECT_STMT: + return jsonToSelectStmt(pJson, pObj); + case QUERY_NODE_CREATE_TOPIC_STMT: + return jsonToCreateTopicStmt(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_SCAN: + return jsonToLogicScanNode(pJson, pObj); // case QUERY_NODE_LOGIC_PLAN_JOIN: // return jsonToLogicJoinNode(pJson, pObj); // case QUERY_NODE_LOGIC_PLAN_AGG: // return jsonToLogicAggNode(pJson, pObj); - // case QUERY_NODE_LOGIC_PLAN_PROJECT: - // return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_PROJECT: + return jsonToLogicProjectNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: return jsonToPhysiTableScanNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + return jsonToPhysiStreamScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - return jsonToPhysiSysTableScanNode(pJson, pObj); + return jsonToPhysiSysTableScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return jsonToPhysiProjectNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_JOIN: @@ -1713,7 +2155,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToSubplan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN: return jsonToPlan(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + return jsonToPhysiIntervalNode(pJson, pObj); default: + assert(0); break; } nodesWarn("jsonToSpecificNode unknown node = %s", nodesNodeName(nodeType(pObj))); @@ -1779,7 +2224,7 @@ static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** p } int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen) { - if (NULL == pNode || NULL == pStr || NULL == pLen) { + if (NULL == pNode || NULL == pStr) { terrno = TSDB_CODE_FAILED; return TSDB_CODE_FAILED; } @@ -1799,7 +2244,10 @@ int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_ *pStr = format ? tjsonToString(pJson) : tjsonToUnformattedString(pJson); tjsonDelete(pJson); - *pLen = strlen(*pStr) + 1; + if (NULL != pLen) { + *pLen = strlen(*pStr) + 1; + } + return TSDB_CODE_SUCCESS; } @@ -1814,6 +2262,56 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode) { int32_t code = makeNodeByJson(pJson, pNode); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode(*pNode); + *pNode = NULL; + terrno = code; + return code; + } + return TSDB_CODE_SUCCESS; +} + +int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen) { + if (NULL == pList || NULL == pStr || NULL == pLen) { + terrno = TSDB_CODE_FAILED; + return TSDB_CODE_FAILED; + } + + if (0 == LIST_LENGTH(pList)) { + return TSDB_CODE_SUCCESS; + } + + SJson* pJson = tjsonCreateArray(); + if (NULL == pJson) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNode* pNode; + FOREACH(pNode, pList) { + int32_t code = tjsonAddItem(pJson, nodeToJson, pNode); + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + return code; + } + } + + *pStr = format ? tjsonToString(pJson) : tjsonToUnformattedString(pJson); + tjsonDelete(pJson); + + *pLen = strlen(*pStr) + 1; + return TSDB_CODE_SUCCESS; +} + +int32_t nodesStringToList(const char* pStr, SNodeList** pList) { + if (NULL == pStr || NULL == pList) { + return TSDB_CODE_SUCCESS; + } + SJson* pJson = tjsonParse(pStr); + if (NULL == pJson) { + return TSDB_CODE_FAILED; + } + int32_t code = jsonToNodeListImpl(pJson, pList); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(*pList); terrno = code; return code; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index b215c29a92676145aac590d32ca67db77cb415f3..14bae19cf1bc1c0b2f9738326e2dbf2a39e12e45 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -76,6 +76,12 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SColumnDefNode)); case QUERY_NODE_DOWNSTREAM_SOURCE: return makeNode(type, sizeof(SDownstreamSourceNode)); + case QUERY_NODE_DATABASE_OPTIONS: + return makeNode(type, sizeof(SDatabaseOptions)); + case QUERY_NODE_TABLE_OPTIONS: + return makeNode(type, sizeof(STableOptions)); + case QUERY_NODE_INDEX_OPTIONS: + return makeNode(type, sizeof(SIndexOptions)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: @@ -86,6 +92,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SCreateDatabaseStmt)); case QUERY_NODE_DROP_DATABASE_STMT: return makeNode(type, sizeof(SDropDatabaseStmt)); + case QUERY_NODE_ALTER_DATABASE_STMT: + return makeNode(type, sizeof(SAlterDatabaseStmt)); case QUERY_NODE_CREATE_TABLE_STMT: return makeNode(type, sizeof(SCreateTableStmt)); case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: @@ -110,6 +118,20 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SCreateDnodeStmt)); case QUERY_NODE_DROP_DNODE_STMT: return makeNode(type, sizeof(SDropDnodeStmt)); + case QUERY_NODE_ALTER_DNODE_STMT: + return makeNode(type, sizeof(SAlterDnodeStmt)); + case QUERY_NODE_CREATE_INDEX_STMT: + return makeNode(type, sizeof(SCreateIndexStmt)); + case QUERY_NODE_DROP_INDEX_STMT: + return makeNode(type, sizeof(SDropIndexStmt)); + case QUERY_NODE_CREATE_QNODE_STMT: + return makeNode(type, sizeof(SCreateQnodeStmt)); + case QUERY_NODE_DROP_QNODE_STMT: + return makeNode(type, sizeof(SDropQnodeStmt)); + case QUERY_NODE_CREATE_TOPIC_STMT: + return makeNode(type, sizeof(SCreateTopicStmt)); + case QUERY_NODE_DROP_TOPIC_STMT: + return makeNode(type, sizeof(SDropTopicStmt)); case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_TABLES_STMT: case QUERY_NODE_SHOW_STABLES_STMT: @@ -148,7 +170,7 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: return makeNode(type, sizeof(STableSeqScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - return makeNode(type, sizeof(SNode)); + return makeNode(type, sizeof(SStreamScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: return makeNode(type, sizeof(SSystemTableScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: @@ -211,6 +233,14 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { case QUERY_NODE_NODE_LIST: nodesClearList(((SNodeListNode*)(*pNode))->pNodeList); break; + case QUERY_NODE_INDEX_OPTIONS: { + SIndexOptions* pStmt = (SIndexOptions*)*pNode; + nodesDestroyList(pStmt->pFuncs); + nodesDestroyNode(pStmt->pInterval); + nodesDestroyNode(pStmt->pOffset); + nodesDestroyNode(pStmt->pSliding); + break; + } case QUERY_NODE_SELECT_STMT: { SSelectStmt* pStmt = (SSelectStmt*)*pNode; nodesDestroyList(pStmt->pProjectionList); @@ -251,6 +281,12 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { case QUERY_NODE_CREATE_MULTI_TABLE_STMT: nodesDestroyList(((SCreateMultiTableStmt*)(*pNode))->pSubTables); break; + case QUERY_NODE_CREATE_INDEX_STMT: { + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)*pNode; + nodesDestroyNode(pStmt->pOptions); + nodesDestroyList(pStmt->pCols); + break; + } default: break; } @@ -325,6 +361,17 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { return TSDB_CODE_SUCCESS; } +int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc) { + if (NULL == pSrc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t code = nodesListAppendList(pTarget, pSrc); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pSrc); + } + return code; +} + SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { if (NULL == pCell->pPrev) { pList->pHead = pCell->pNext; @@ -539,7 +586,7 @@ typedef struct SCollectFuncsCxt { static EDealRes collectFuncs(SNode* pNode, void* pContext) { SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext; if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId)) { - pCxt->errCode = nodesListAppend(pCxt->pFuncs, pNode); + pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode)); return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } return DEAL_RES_CONTINUE; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 7ecfe626cec879d7d0db3babfbb1fa004e74e010..45216be6ef0785241e04c5277394b58492b8c9ca 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -50,8 +50,10 @@ typedef enum EDatabaseOptionType { DB_OPTION_TTL, DB_OPTION_WAL, DB_OPTION_VGROUPS, - DB_OPTION_SINGLESTABLE, - DB_OPTION_STREAMMODE, + DB_OPTION_SINGLE_STABLE, + DB_OPTION_STREAM_MODE, + DB_OPTION_RETENTIONS, + DB_OPTION_FILE_FACTOR, DB_OPTION_MAX } EDatabaseOptionType; @@ -65,6 +67,11 @@ typedef enum ETableOptionType { TABLE_OPTION_MAX } ETableOptionType; +typedef struct SAlterOption { + int32_t type; + SToken val; +} SAlterOption; + extern SToken nil_token; void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt); @@ -110,22 +117,31 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); -SDatabaseOptions* createDefaultDatabaseOptions(SAstCreateContext* pCxt); -SDatabaseOptions* setDatabaseOption(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, EDatabaseOptionType type, const SToken* pVal); -SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SDatabaseOptions* pOptions); +SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt); +SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt); +SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal); +SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions); SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName); -STableOptions* createDefaultTableOptions(SAstCreateContext* pCxt); -STableOptions* setTableOption(SAstCreateContext* pCxt, STableOptions* pOptions, ETableOptionType type, const SToken* pVal); -STableOptions* setTableSmaOption(SAstCreateContext* pCxt, STableOptions* pOptions, SNodeList* pSma); +SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions); +SNode* createDefaultTableOptions(SAstCreateContext* pCxt); +SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt); +SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal); +SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma); +SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs); SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment); SDataType createDataType(uint8_t type); SDataType createVarLenDataType(uint8_t type, const SToken* pLen); -SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNodeList* pCols, SNodeList* pTags, STableOptions* pOptions); +SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNodeList* pCols, SNodeList* pTags, SNode* pOptions); SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SNode* pRealTable, SNode* pUseRealTable, SNodeList* pSpecificTags, SNodeList* pValsOfTags); SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables); SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables); SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); +SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions); +SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType); +SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName); +SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName); +SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal); SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName); SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbNamePattern); SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword); @@ -133,6 +149,15 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, int SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode); +SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue); +SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions); +SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding); +SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName); +SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); +SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); +SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName); +SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName); +SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 3f2f897289e7f97ee60f75de78fd3b8b5909c35f..563d3213dab4764d2d3763c5f290f3b0d41288ef 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -21,13 +21,15 @@ #include "parAst.h" } -%syntax_error { - if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); - } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); +%syntax_error { + if (pCxt->valid) { + if(TOKEN.z) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); + } else { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + } + pCxt->valid = false; } - pCxt->valid = false; } %left OR. @@ -41,17 +43,56 @@ %left NK_CONCAT. //%right NK_BITNOT. -/************************************************ create/alter/drop user **********************************************/ -cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B);} -cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B);} -cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B);} +/************************************************ create/alter account *****************************************/ +cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } + +%type account_options { int32_t } +%destructor account_options { } +account_options ::= . { } +account_options ::= account_options PPS literal. { } +account_options ::= account_options TSERIES literal. { } +account_options ::= account_options STORAGE literal. { } +account_options ::= account_options STREAMS literal. { } +account_options ::= account_options QTIME literal. { } +account_options ::= account_options DBS literal. { } +account_options ::= account_options USERS literal. { } +account_options ::= account_options CONNS literal. { } +account_options ::= account_options STATE literal. { } + +%type alter_account_options { int32_t } +%destructor alter_account_options { } +alter_account_options ::= alter_account_option. { } +alter_account_options ::= alter_account_options alter_account_option. { } + +%type alter_account_option { int32_t } +%destructor alter_account_option { } +alter_account_option ::= PASS literal. { } +alter_account_option ::= PPS literal. { } +alter_account_option ::= TSERIES literal. { } +alter_account_option ::= STORAGE literal. { } +alter_account_option ::= STREAMS literal. { } +alter_account_option ::= QTIME literal. { } +alter_account_option ::= DBS literal. { } +alter_account_option ::= USERS literal. { } +alter_account_option ::= CONNS literal. { } +alter_account_option ::= STATE literal. { } + +/************************************************ create/alter/drop/show user *****************************************/ +cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); } +cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); } +cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); } cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } -/************************************************ create/drop dnode ***************************************************/ -cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL);} -cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B);} -cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} -cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A);} +/************************************************ create/drop/alter/show dnode ****************************************/ +cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } +cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); } +cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } +cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } +cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, NULL); } +cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B) NK_STRING(C). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, &C); } +cmd ::= ALTER ALL DNODES NK_STRING(A). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, NULL); } +cmd ::= ALTER ALL DNODES NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, &B); } %type dnode_endpoint { SToken } %destructor dnode_endpoint { } @@ -62,10 +103,19 @@ dnode_endpoint(A) ::= NK_STRING(B). dnode_host_name(A) ::= NK_ID(B). { A = B; } dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; } -/************************************************ create/drop/use database ********************************************/ -cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C). { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, A, &B, C);} +/************************************************ alter local *********************************************************/ +cmd ::= ALTER LOCAL NK_STRING(A). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, NULL); } +cmd ::= ALTER LOCAL NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, &B); } + +/************************************************ create/drop qnode ***************************************************/ +cmd ::= CREATE QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &A); } +cmd ::= DROP QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropQnodeStmt(pCxt, &A); } + +/************************************************ create/drop/show/use database ***************************************/ +cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C). { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, A, &B, C); } cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } -cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A);} +cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); } +cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } %type not_exists_opt { bool } %destructor not_exists_opt { } @@ -77,8 +127,6 @@ not_exists_opt(A) ::= . exists_opt(A) ::= IF EXISTS. { A = true; } exists_opt(A) ::= . { A = false; } -%type db_options { SDatabaseOptions* } -%destructor db_options { tfree($$); } db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } db_options(A) ::= db_options(B) BLOCKS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BLOCKS, &C); } db_options(A) ::= db_options(B) CACHE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHE, &C); } @@ -95,18 +143,53 @@ db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } -db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); } -db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); } +db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); } +db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); } +db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); } +db_options(A) ::= db_options(B) FILE_FACTOR NK_FLOAT(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FILE_FACTOR, &C); } + +alter_db_options(A) ::= alter_db_option(B). { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); } +alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseOption(pCxt, B, C.type, &C.val); } + +%type alter_db_option { SAlterOption } +%destructor alter_db_option { } +alter_db_option(A) ::= BLOCKS NK_INTEGER(B). { A.type = DB_OPTION_BLOCKS; A.val = B; } +alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; } +alter_db_option(A) ::= KEEP NK_INTEGER(B). { A.type = DB_OPTION_KEEP; A.val = B; } +alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; } +alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.val = B; } +alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; } /************************************************ create/drop table/stable ********************************************/ cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) - NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} -cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A);} + NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } +cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A); } cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) - NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} + NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); } +cmd ::= ALTER TABLE alter_table_clause(A). { pCxt->pRootNode = A; } +cmd ::= ALTER STABLE alter_table_clause(A). { pCxt->pRootNode = A; } + +alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableOption(pCxt, B, C); } +alter_table_clause(A) ::= + full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); } +alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); } +alter_table_clause(A) ::= + full_table_name(B) MODIFY COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &C, D); } +alter_table_clause(A) ::= + full_table_name(B) RENAME COLUMN column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &C, &D); } +alter_table_clause(A) ::= + full_table_name(B) ADD TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_TAG, &C, D); } +alter_table_clause(A) ::= full_table_name(B) DROP TAG column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_TAG, &C); } +alter_table_clause(A) ::= + full_table_name(B) MODIFY TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &C, D); } +alter_table_clause(A) ::= + full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); } +alter_table_clause(A) ::= + full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, D); } + %type multi_create_clause { SNodeList* } %destructor multi_create_clause { nodesDestroyList($$); } multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } @@ -174,13 +257,21 @@ tags_def_opt(A) ::= tags_def(B). %destructor tags_def { nodesDestroyList($$); } tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } -%type table_options { STableOptions* } -%destructor table_options { tfree($$); } -table_options(A) ::= . { A = createDefaultTableOptions(pCxt);} +table_options(A) ::= . { A = createDefaultTableOptions(pCxt); } table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } +table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableRollupOption(pCxt, B, C); } + +alter_table_options(A) ::= alter_table_option(B). { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); } +alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); } + +%type alter_table_option { SAlterOption } +%destructor alter_table_option { } +alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.val = B; } +alter_table_option(A) ::= KEEP NK_INTEGER(B). { A.type = TABLE_OPTION_KEEP; A.val = B; } +alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.val = B; } %type col_name_list { SNodeList* } %destructor col_name_list { nodesDestroyList($$); } @@ -189,7 +280,6 @@ col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C). col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } -<<<<<<< HEAD /************************************************ show ****************************************************************/ cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL, NULL); } cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL, NULL); } @@ -215,6 +305,37 @@ table_name_cond(A) ::= table_name(B). from_db_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } from_db_opt(A) ::= FROM db_name(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } +%type func_name_list { SNodeList* } +%destructor func_name_list { nodesDestroyList($$); } +func_name_list(A) ::= func_name(B). { A = createNodeList(pCxt, B); } +func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); } + +func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); } + +/************************************************ create index ********************************************************/ +cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); } +cmd ::= CREATE FULLTEXT INDEX + index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &A, &B, C, NULL); } +cmd ::= DROP INDEX index_name(A) ON table_name(B). { pCxt->pRootNode = createDropIndexStmt(pCxt, &A, &B); } + +index_options(A) ::= . { A = NULL; } +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL + NK_LP duration_literal(C) NK_RP sliding_opt(D). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), NULL, D); } +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL + NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D), E); } + +%type func_list { SNodeList* } +%destructor func_list { nodesDestroyList($$); } +func_list(A) ::= func(B). { A = createNodeList(pCxt, B); } +func_list(A) ::= func_list(B) NK_COMMA func(C). { A = addNodeToList(pCxt, B, C); } + +func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } + +/************************************************ create/drop topic ***************************************************/ +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, C, NULL); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS db_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C); } +cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); } + /************************************************ select **************************************************************/ cmd ::= query_expression(A). { pCxt->pRootNode = A; } @@ -262,6 +383,14 @@ column_alias(A) ::= NK_ID(B). %destructor user_name { } user_name(A) ::= NK_ID(B). { A = B; } +%type index_name { SToken } +%destructor index_name { } +index_name(A) ::= NK_ID(B). { A = B; } + +%type topic_name { SToken } +%destructor topic_name { } +topic_name(A) ::= NK_ID(B). { A = B; } + /************************************************ expression **********************************************************/ expression(A) ::= literal(B). { A = B; } //expression(A) ::= NK_QUESTION(B). { A = B; } @@ -477,13 +606,13 @@ twindow_clause_opt(A) ::= SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); } twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } twindow_clause_opt(A) ::= - INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, B, NULL, C, D); } + INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); } twindow_clause_opt(A) ::= INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP - sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, B, C, D, E); } + sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), D, E); } sliding_opt(A) ::= . { A = NULL; } -sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = B; } +sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } fill_opt(A) ::= . { A = NULL; } fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } @@ -526,7 +655,7 @@ query_expression_body(A) ::= query_primary(A) ::= query_specification(B). { A = B; } //query_primary(A) ::= // NK_LP query_expression_body(B) -// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B;} +// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B; } %type order_by_clause_opt { SNodeList* } %destructor order_by_clause_opt { nodesDestroyList($$); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index cd6ddd182c3190811a7a8382f19db7562b141032..bad87a29b3cc8b32d90859e2dcbbff8b82c4625e 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -247,6 +247,16 @@ static SDatabaseOptions* setDbStreamMode(SAstCreateContext* pCxt, SDatabaseOptio return pOptions; } +static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) { + // todo + return pOptions; +} + +static SDatabaseOptions* setDbFileFactor(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) { + // todo + return pOptions; +} + static void initSetDatabaseOptionFp() { setDbOptionFuncs[DB_OPTION_BLOCKS] = setDbBlocks; setDbOptionFuncs[DB_OPTION_CACHE] = setDbCache; @@ -263,8 +273,10 @@ static void initSetDatabaseOptionFp() { setDbOptionFuncs[DB_OPTION_TTL] = setDbTtl; setDbOptionFuncs[DB_OPTION_WAL] = setDbWal; setDbOptionFuncs[DB_OPTION_VGROUPS] = setDbVgroups; - setDbOptionFuncs[DB_OPTION_SINGLESTABLE] = setDbSingleStable; - setDbOptionFuncs[DB_OPTION_STREAMMODE] = setDbStreamMode; + setDbOptionFuncs[DB_OPTION_SINGLE_STABLE] = setDbSingleStable; + setDbOptionFuncs[DB_OPTION_STREAM_MODE] = setDbStreamMode; + setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions; + setDbOptionFuncs[DB_OPTION_FILE_FACTOR] = setDbFileFactor; } static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) { @@ -427,6 +439,14 @@ static bool checkColumnName(SAstCreateContext* pCxt, const SToken* pColumnName) return pCxt->valid; } +static bool checkIndexName(SAstCreateContext* pCxt, const SToken* pIndexName) { + if (NULL == pIndexName) { + return false; + } + pCxt->valid = pIndexName->n < TSDB_INDEX_NAME_LEN ? true : false; + return pCxt->valid; +} + SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode) { SRawExprNode* target = (SRawExprNode*)nodesMakeNode(QUERY_NODE_RAW_EXPR); CHECK_OUT_OF_MEM(target); @@ -769,8 +789,8 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* return (SNode*)setOp; } -SDatabaseOptions* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { - SDatabaseOptions* pOptions = calloc(1, sizeof(SDatabaseOptions)); +SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { + SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); pOptions->numOfBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; pOptions->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; @@ -789,14 +809,37 @@ SDatabaseOptions* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; pOptions->singleStable = TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION; pOptions->streamMode = TSDB_DEFAULT_DB_STREAM_MODE_OPTION; - return pOptions; -} - -SDatabaseOptions* setDatabaseOption(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, EDatabaseOptionType type, const SToken* pVal) { - return setDbOptionFuncs[type](pCxt, pOptions, pVal); + return (SNode*)pOptions; } -SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SDatabaseOptions* pOptions) { +SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt) { + SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); + CHECK_OUT_OF_MEM(pOptions); + pOptions->numOfBlocks = -1; + pOptions->cacheBlockSize = -1; + pOptions->cachelast = -1; + pOptions->compressionLevel = -1; + pOptions->daysPerFile = -1; + pOptions->fsyncPeriod = -1; + pOptions->maxRowsPerBlock = -1; + pOptions->minRowsPerBlock = -1; + pOptions->keep = -1; + pOptions->precision = -1; + pOptions->quorum = -1; + pOptions->replica = -1; + pOptions->ttl = -1; + pOptions->walLevel = -1; + pOptions->numOfVgroups = -1; + pOptions->singleStable = -1; + pOptions->streamMode = -1; + return (SNode*)pOptions; +} + +SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal) { + return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal); +} + +SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions) { if (!checkDbName(pCxt, pDbName, false)) { return NULL; } @@ -804,8 +847,7 @@ SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, cons CHECK_OUT_OF_MEM(pStmt); strncpy(pStmt->dbName, pDbName->z, pDbName->n); pStmt->ignoreExists = ignoreExists; - pStmt->options = *pOptions; - tfree(pOptions); + pStmt->pOptions = (SDatabaseOptions*)pOptions; return (SNode*)pStmt; } @@ -820,20 +862,44 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, con return (SNode*)pStmt; } -STableOptions* createDefaultTableOptions(SAstCreateContext* pCxt) { - STableOptions* pOptions = calloc(1, sizeof(STableOptions)); +SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions) { + if (!checkDbName(pCxt, pDbName, false)) { + return NULL; + } + SAlterDatabaseStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DATABASE_STMT); + CHECK_OUT_OF_MEM(pStmt); + strncpy(pStmt->dbName, pDbName->z, pDbName->n); + pStmt->pOptions = (SDatabaseOptions*)pOptions; + return (SNode*)pStmt; +} + +SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { + STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); pOptions->keep = TSDB_DEFAULT_KEEP; pOptions->ttl = TSDB_DEFAULT_DB_TTL_OPTION; - return pOptions; + return (SNode*)pOptions; } -STableOptions* setTableOption(SAstCreateContext* pCxt, STableOptions* pOptions, ETableOptionType type, const SToken* pVal) { - return setTableOptionFuncs[type](pCxt, pOptions, pVal); +SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) { + STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); + CHECK_OUT_OF_MEM(pOptions); + pOptions->keep = -1; + pOptions->ttl = -1; + return (SNode*)pOptions; } -STableOptions* setTableSmaOption(SAstCreateContext* pCxt, STableOptions* pOptions, SNodeList* pSma) { - pOptions->pSma = pSma; +SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal) { + return (SNode*)setTableOptionFuncs[type](pCxt, (STableOptions*)pOptions, pVal); +} + +SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma) { + ((STableOptions*)pOptions)->pSma = pSma; + return pOptions; +} + +SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs) { + // todo return pOptions; } @@ -859,7 +925,7 @@ SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { } SNode* createCreateTableStmt(SAstCreateContext* pCxt, - bool ignoreExists, SNode* pRealTable, SNodeList* pCols, SNodeList* pTags, STableOptions* pOptions) { + bool ignoreExists, SNode* pRealTable, SNodeList* pCols, SNodeList* pTags, SNode* pOptions) { SCreateTableStmt* pStmt = (SCreateTableStmt*)nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); CHECK_OUT_OF_MEM(pStmt); strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); @@ -867,9 +933,7 @@ SNode* createCreateTableStmt(SAstCreateContext* pCxt, pStmt->ignoreExists = ignoreExists; pStmt->pCols = pCols; pStmt->pTags = pTags; - pStmt->options = *pOptions; - nodesDestroyList(pOptions->pSma); - tfree(pOptions); + pStmt->pOptions = (STableOptions*)pOptions; nodesDestroyNode(pRealTable); return (SNode*)pStmt; } @@ -924,6 +988,49 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S return (SNode*)pStmt; } +SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_OPTIONS; + pStmt->pOptions = (STableOptions*)pOptions; + return (SNode*)pStmt; +} + +SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = alterType; + strncpy(pStmt->colName, pColName->z, pColName->n); + pStmt->dataType = dataType; + return (SNode*)pStmt; +} + +SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = alterType; + strncpy(pStmt->colName, pColName->z, pColName->n); + return (SNode*)pStmt; +} + +SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = alterType; + strncpy(pStmt->colName, pOldColName->z, pOldColName->n); + strncpy(pStmt->newColName, pNewColName->z, pNewColName->n); + return (SNode*)pStmt; +} + +SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal) { + SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; + strncpy(pStmt->colName, pTagName->z, pTagName->n); + pStmt->pVal = (SValueNode*)pVal; + return (SNode*)pStmt; +} + SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) { SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT); CHECK_OUT_OF_MEM(pStmt); @@ -1023,3 +1130,93 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) { } return (SNode*)pStmt; } + +SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue) { + SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); + if (NULL != pValue) { + trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); + } + return (SNode*)pStmt; +} + +SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions) { + if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { + return NULL; + } + SCreateIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_INDEX_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->indexType = type; + strncpy(pStmt->indexName, pIndexName->z, pIndexName->n); + strncpy(pStmt->tableName, pTableName->z, pTableName->n); + pStmt->pCols = pCols; + pStmt->pOptions = (SIndexOptions*)pOptions; + return (SNode*)pStmt; +} + +SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding) { + SIndexOptions* pOptions = nodesMakeNode(QUERY_NODE_INDEX_OPTIONS); + CHECK_OUT_OF_MEM(pOptions); + pOptions->pFuncs = pFuncs; + pOptions->pInterval = pInterval; + pOptions->pOffset = pOffset; + pOptions->pSliding = pSliding; + return (SNode*)pOptions; +} + +SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName) { + if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { + return NULL; + } + SDropIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT); + CHECK_OUT_OF_MEM(pStmt); + strncpy(pStmt->indexName, pIndexName->z, pIndexName->n); + strncpy(pStmt->tableName, pTableName->z, pTableName->n); + return (SNode*)pStmt; +} + +SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId) { + SCreateQnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_QNODE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10);; + return (SNode*)pStmt; +} + +SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId) { + SDropQnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_QNODE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10);; + return (SNode*)pStmt; +} + +SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName) { + SCreateTopicStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); + CHECK_OUT_OF_MEM(pStmt); + strncpy(pStmt->topicName, pTopicName->z, pTopicName->n); + pStmt->ignoreExists = ignoreExists; + pStmt->pQuery = pQuery; + if (NULL != pSubscribeDbName) { + strncpy(pStmt->subscribeDbName, pSubscribeDbName->z, pSubscribeDbName->n); + } + return (SNode*)pStmt; +} + +SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName) { + SDropTopicStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_TOPIC_STMT); + CHECK_OUT_OF_MEM(pStmt); + strncpy(pStmt->topicName, pTopicName->z, pTopicName->n); + pStmt->ignoreNotExists = ignoreNotExists; + return (SNode*)pStmt; +} + +SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) { + SAlterLocalStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT); + CHECK_OUT_OF_MEM(pStmt); + trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); + if (NULL != pValue) { + trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); + } + return (SNode*)pStmt; +} diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index e1b9fce5e803a2b029a42a70088e6708d495543b..a27a5454eb558161466b2db8df7b66e1a974e00e 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -396,7 +396,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha } // Remove quotation marks - if (TSDB_DATA_TYPE_BINARY == type) { + if (TK_NK_STRING == pToken->type) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } @@ -622,18 +622,18 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { const char* rowEnd = tdRowEnd(rb->pBuf); STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, true, pa->toffset, pa->colIdx); } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; const char* rowEnd = tdRowEnd(rb->pBuf); - if (!taosMbsToUcs4(value, len, (char*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } varDataSetLen(rowEnd, output); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, true, pa->toffset, pa->colIdx); } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, true, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); } return TSDB_CODE_SUCCESS; } @@ -730,7 +730,7 @@ static int32_t KvRowAppend(const void *value, int32_t len, void *param) { } else if (TSDB_DATA_TYPE_NCHAR == type) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; - if (!taosMbsToUcs4(value, len, varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } @@ -768,6 +768,8 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, // todo construct payload tfree(row); + + return 0; } // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 0617a399c4fd9a2ebd62c50f4064091626cdd8d5..a305aa56e1b23419dcc910b687a2a96d9ab50f1e 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -28,6 +28,7 @@ typedef struct SKeyword { // keywords in sql string static SKeyword keywordTable[] = { + {"ACCOUNT", TK_ACCOUNT}, {"ALL", TK_ALL}, {"ALTER", TK_ALTER}, {"AND", TK_AND}, @@ -59,12 +60,14 @@ static SKeyword keywordTable[] = { {"FLOAT", TK_FLOAT}, {"FROM", TK_FROM}, {"FSYNC", TK_FSYNC}, + {"FUNCTION", TK_FUNCTION}, {"FUNCTIONS", TK_FUNCTIONS}, {"GROUP", TK_GROUP}, {"HAVING", TK_HAVING}, {"IF", TK_IF}, {"IMPORT", TK_IMPORT}, {"IN", TK_IN}, + {"INDEX", TK_INDEX}, {"INDEXES", TK_INDEXES}, {"INNER", TK_INNER}, {"INT", TK_INT}, @@ -100,6 +103,7 @@ static SKeyword keywordTable[] = { {"PRECISION", TK_PRECISION}, {"PRIVILEGE", TK_PRIVILEGE}, {"PREV", TK_PREV}, + {"QNODE", TK_QNODE}, {"QNODES", TK_QNODES}, {"QUORUM", TK_QUORUM}, {"REPLICA", TK_REPLICA}, @@ -122,6 +126,7 @@ static SKeyword keywordTable[] = { {"TAGS", TK_TAGS}, {"TIMESTAMP", TK_TIMESTAMP}, {"TINYINT", TK_TINYINT}, + {"TOPIC", TK_TOPIC}, {"TTL", TK_TTL}, {"UNION", TK_UNION}, {"UNSIGNED", TK_UNSIGNED}, @@ -166,7 +171,6 @@ static SKeyword keywordTable[] = { // {"SCORES", TK_SCORES}, // {"GRANTS", TK_GRANTS}, // {"DOT", TK_DOT}, - // {"ACCOUNT", TK_ACCOUNT}, // {"DESCRIBE", TK_DESCRIBE}, // {"SYNCDB", TK_SYNCDB}, // {"LOCAL", TK_LOCAL}, @@ -229,11 +233,9 @@ static SKeyword keywordTable[] = { // {"TBNAME", TK_TBNAME}, // {"VNODES", TK_VNODES}, // {"PARTITIONS", TK_PARTITIONS}, - // {"TOPIC", TK_TOPIC}, // {"TOPICS", TK_TOPICS}, // {"COMPACT", TK_COMPACT}, // {"MODIFY", TK_MODIFY}, - // {"FUNCTION", TK_FUNCTION}, // {"OUTPUTTYPE", TK_OUTPUTTYPE}, // {"AGGREGATE", TK_AGGREGATE}, // {"BUFSIZE", TK_BUFSIZE}, @@ -265,10 +267,10 @@ static void doInitKeywordsTable(void) { } } -static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT; +static TdThreadOnce keywordsHashTableInit = PTHREAD_ONCE_INIT; static int32_t tKeywordCode(const char* z, int n) { - pthread_once(&keywordsHashTableInit, doInitKeywordsTable); + taosThreadOnce(&keywordsHashTableInit, doInitKeywordsTable); char key[512] = {0}; if (n > tListLen(key)) { // too long token, can not be any other token type diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fb796aef81b8ae43973367d4e5de97bcdecd5cda..5211fdc682fe8862497b9d464d49e5a75d1c22b4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -21,14 +21,6 @@ #include "parUtil.h" #include "ttime.h" -static bool afterGroupBy(ESqlClause clause) { - return clause > SQL_CLAUSE_GROUP_BY; -} - -static bool beforeHaving(ESqlClause clause) { - return clause < SQL_CLAUSE_HAVING; -} - typedef struct STranslateContext { SParseContext* pParseCxt; int32_t errCode; @@ -41,6 +33,15 @@ typedef struct STranslateContext { } STranslateContext; static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); +static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode); + +static bool afterGroupBy(ESqlClause clause) { + return clause > SQL_CLAUSE_GROUP_BY; +} + +static bool beforeHaving(ESqlClause clause) { + return clause < SQL_CLAUSE_HAVING; +} static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) { va_list vArgList; @@ -189,7 +190,7 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; - int32_t nums = pMeta->tableInfo.numOfColumns + ((TSDB_SUPER_TABLE == pMeta->tableType)? pMeta->tableInfo.numOfTags:0); + int32_t nums = pMeta->tableInfo.numOfColumns + ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0); for (int32_t i = 0; i < nums; ++i) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { @@ -309,8 +310,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { if (pVal->isDuration) { - char unit = 0; - if (parseAbsoluteDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) { + if (parseAbsoluteDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } } else { @@ -587,6 +587,10 @@ static int32_t setSysTableVgroupList(SParseContext* pCxt, SName* pName, SRealTab } static int32_t setTableVgroupList(SParseContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + if (pCxt->topicQuery) { + return TSDB_CODE_SUCCESS; + } + int32_t code = TSDB_CODE_SUCCESS; if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { SArray* vgroupList = NULL; @@ -844,26 +848,26 @@ static void buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameGetFullDbName(&name, pReq->db); - pReq->numOfVgroups = pStmt->options.numOfVgroups; - pReq->cacheBlockSize = pStmt->options.cacheBlockSize; - pReq->totalBlocks = pStmt->options.numOfBlocks; - pReq->daysPerFile = pStmt->options.daysPerFile; - pReq->daysToKeep0 = pStmt->options.keep; + pReq->numOfVgroups = pStmt->pOptions->numOfVgroups; + pReq->cacheBlockSize = pStmt->pOptions->cacheBlockSize; + pReq->totalBlocks = pStmt->pOptions->numOfBlocks; + pReq->daysPerFile = pStmt->pOptions->daysPerFile; + pReq->daysToKeep0 = pStmt->pOptions->keep; pReq->daysToKeep1 = -1; pReq->daysToKeep2 = -1; - pReq->minRows = pStmt->options.minRowsPerBlock; - pReq->maxRows = pStmt->options.maxRowsPerBlock; + pReq->minRows = pStmt->pOptions->minRowsPerBlock; + pReq->maxRows = pStmt->pOptions->maxRowsPerBlock; pReq->commitTime = -1; - pReq->fsyncPeriod = pStmt->options.fsyncPeriod; - pReq->walLevel = pStmt->options.walLevel; - pReq->precision = pStmt->options.precision; - pReq->compression = pStmt->options.compressionLevel; - pReq->replications = pStmt->options.replica; - pReq->quorum = pStmt->options.quorum; + pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; + pReq->walLevel = pStmt->pOptions->walLevel; + pReq->precision = pStmt->pOptions->precision; + pReq->compression = pStmt->pOptions->compressionLevel; + pReq->replications = pStmt->pOptions->replica; + pReq->quorum = pStmt->pOptions->quorum; pReq->update = -1; - pReq->cacheLastRow = pStmt->options.cachelast; + pReq->cacheLastRow = pStmt->pOptions->cachelast; pReq->ignoreExist = pStmt->ignoreExists; - pReq->streamMode = pStmt->options.streamMode; + pReq->streamMode = pStmt->pOptions->streamMode; return; } @@ -872,14 +876,14 @@ static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseS buildCreateDbReq(pCxt, pStmt, &createReq); pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DB; pCxt->pCmdMsg->msgLen = tSerializeSCreateDbReq(NULL, 0, &createReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSCreateDbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createReq); @@ -895,14 +899,14 @@ static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* dropReq.ignoreNotExists = pStmt->ignoreNotExists; pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_DB; pCxt->pCmdMsg->msgLen = tSerializeSDropDbReq(NULL, 0, &dropReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSDropDbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &dropReq); @@ -910,6 +914,41 @@ static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* return TSDB_CODE_SUCCESS; } +static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, SAlterDbReq* pReq) { + SName name = {0}; + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + tNameGetFullDbName(&name, pReq->db); + pReq->totalBlocks = pStmt->pOptions->numOfBlocks; + pReq->daysToKeep0 = pStmt->pOptions->keep; + pReq->daysToKeep1 = -1; + pReq->daysToKeep2 = -1; + pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; + pReq->walLevel = pStmt->pOptions->walLevel; + pReq->quorum = pStmt->pOptions->quorum; + pReq->cacheLastRow = pStmt->pOptions->cachelast; + return; +} + +static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { + SAlterDbReq alterReq = {0}; + buildAlterDbReq(pCxt, pStmt, &alterReq); + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_DB; + pCxt->pCmdMsg->msgLen = tSerializeSAlterDbReq(NULL, 0, &alterReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSAlterDbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &alterReq); + + return TSDB_CODE_SUCCESS; +} + static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) { *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); SNode* pNode; @@ -936,7 +975,7 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt tNameExtractFullName(&tableName, createReq.name); pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { tFreeSMCreateStbReq(&createReq); return TSDB_CODE_OUT_OF_MEMORY; } @@ -944,7 +983,7 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_STB; pCxt->pCmdMsg->msgLen = tSerializeSMCreateStbReq(NULL, 0, &createReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { tFreeSMCreateStbReq(&createReq); return TSDB_CODE_OUT_OF_MEMORY; } @@ -960,14 +999,14 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p dropReq.igNotExists = ignoreNotExists; pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_STB; pCxt->pCmdMsg->msgLen = tSerializeSMDropStbReq(NULL, 0, &dropReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSMDropStbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &dropReq); @@ -1002,6 +1041,73 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); } +static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) { + pAlterReq->pFields = taosArrayInit(2, sizeof(TAOS_FIELD)); + if (NULL == pAlterReq->pFields) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + switch (pStmt->alterType) { + case TSDB_ALTER_TABLE_ADD_TAG: + case TSDB_ALTER_TABLE_DROP_TAG: + case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_DROP_COLUMN: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: { + TAOS_FIELD field = { .type = pStmt->dataType.type, .bytes = pStmt->dataType.bytes }; + strcpy(field.name, pStmt->colName); + taosArrayPush(pAlterReq->pFields, &field); + break; + } + case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { + TAOS_FIELD oldField = {0}; + strcpy(oldField.name, pStmt->colName); + taosArrayPush(pAlterReq->pFields, &oldField); + TAOS_FIELD newField = {0}; + strcpy(oldField.name, pStmt->newColName); + taosArrayPush(pAlterReq->pFields, &newField); + break; + } + default: + break; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) { + SMAltertbReq alterReq = {0}; + SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; + strcpy(tableName.dbname, pStmt->dbName); + strcpy(tableName.tname, pStmt->tableName); + tNameExtractFullName(&tableName, alterReq.name); + alterReq.alterType = pStmt->alterType; + alterReq.numOfFields = 1; + if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) { + // todo + } else { + if (TSDB_CODE_SUCCESS != setAlterTableField(pStmt, &alterReq)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_STB; + pCxt->pCmdMsg->msgLen = tSerializeSMAlterStbReq(NULL, 0, &alterReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSMAlterStbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &alterReq); + + return TSDB_CODE_SUCCESS; +} + static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) { SUseDbReq usedbReq = {0}; SName name = {0}; @@ -1013,14 +1119,14 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p } pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_USE_DB; pCxt->pCmdMsg->msgLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSUseDbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &usedbReq); @@ -1036,14 +1142,14 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt strcpy(createReq.pass, pStmt->password); pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_USER; pCxt->pCmdMsg->msgLen = tSerializeSCreateUserReq(NULL, 0, &createReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSCreateUserReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createReq); @@ -1062,14 +1168,14 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt } pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_USER; pCxt->pCmdMsg->msgLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSAlterUserReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &alterReq); @@ -1082,14 +1188,14 @@ static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) strcpy(dropReq.user, pStmt->useName); pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_USER; pCxt->pCmdMsg->msgLen = tSerializeSDropUserReq(NULL, 0, &dropReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSDropUserReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &dropReq); @@ -1103,14 +1209,14 @@ static int32_t translateCreateDnode(STranslateContext* pCxt, SCreateDnodeStmt* p createReq.port = pStmt->port; pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSCreateDnodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createReq); @@ -1125,14 +1231,14 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt dropReq.port = pStmt->port; pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSDropDnodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &dropReq); @@ -1140,6 +1246,28 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt return TSDB_CODE_SUCCESS; } +static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) { + SMCfgDnodeReq cfgReq = {0}; + cfgReq.dnodeId = pStmt->dnodeId; + strcpy(cfgReq.config, pStmt->config); + strcpy(cfgReq.value, pStmt->value); + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_MND_CONFIG_DNODE; + pCxt->pCmdMsg->msgLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSMCfgDnodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &cfgReq); + + return TSDB_CODE_SUCCESS; +} + static int32_t nodeTypeToShowType(ENodeType nt) { switch (nt) { case QUERY_NODE_SHOW_DATABASES_STMT: @@ -1154,6 +1282,8 @@ static int32_t nodeTypeToShowType(ENodeType nt) { return TSDB_MGMT_TABLE_VGROUP; case QUERY_NODE_SHOW_MNODES_STMT: return TSDB_MGMT_TABLE_MNODE; + case QUERY_NODE_SHOW_QNODES_STMT: + return TSDB_MGMT_TABLE_QNODE; default: break; } @@ -1170,14 +1300,14 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) { // } pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_SHOW; pCxt->pCmdMsg->msgLen = tSerializeSShowReq(NULL, 0, &showReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); - if (NULL== pCxt->pCmdMsg->pMsg) { + if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } tSerializeSShowReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &showReq); @@ -1197,7 +1327,7 @@ static int32_t translateShowTables(STranslateContext* pCxt) { pShowReq->head.vgId = htonl(info->vgId); pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); - if (NULL== pCxt->pCmdMsg) { + if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = info->epSet; @@ -1209,6 +1339,198 @@ static int32_t translateShowTables(STranslateContext* pCxt) { return TSDB_CODE_SUCCESS; } +static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + SVCreateTSmaReq createSmaReq = {0}; + + if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval) || + (NULL != pStmt->pOptions->pOffset && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset)) || + (NULL != pStmt->pOptions->pSliding && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding))) { + return pCxt->errCode; + } + + createSmaReq.tSma.intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; + createSmaReq.tSma.slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : 0); + strcpy(createSmaReq.tSma.indexName, pStmt->indexName); + + SName name; + name.type = TSDB_TABLE_NAME_T; + name.acctId = pCxt->pParseCxt->acctId; + strcpy(name.dbname, pCxt->pParseCxt->db); + strcpy(name.tname, pStmt->tableName); + STableMeta* pMeta = NULL; + int32_t code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &pCxt->pParseCxt->mgmtEpSet, &name, &pMeta); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + createSmaReq.tSma.tableUid = pMeta->uid; + createSmaReq.tSma.interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; + createSmaReq.tSma.sliding = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : 0); + code = nodesListToString(pStmt->pOptions->pFuncs, false, &createSmaReq.tSma.expr, &createSmaReq.tSma.exprLen); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_VND_CREATE_SMA; + pCxt->pCmdMsg->msgLen = tSerializeSVCreateTSmaReq(NULL, &createSmaReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + void* pBuf = pCxt->pCmdMsg->pMsg; + tSerializeSVCreateTSmaReq(&pBuf, &createSmaReq); + tdDestroyTSma(&createSmaReq.tSma); + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + if (INDEX_TYPE_SMA == pStmt->indexType) { + return translateCreateSmaIndex(pCxt, pStmt); + } else { + // todo fulltext index + return TSDB_CODE_FAILED; + } +} + +static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { + SVDropTSmaReq dropSmaReq = {0}; + strcpy(dropSmaReq.indexName, pStmt->indexName); + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA; + pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + void* pBuf = pCxt->pCmdMsg->pMsg; + tSerializeSVDropTSmaReq(&pBuf, &dropSmaReq); + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* pStmt) { + SMCreateQnodeReq createReq = { .dnodeId = pStmt->dnodeId }; + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_DND_CREATE_QNODE; + pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSMCreateDropQSBNodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createReq); + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateDropQnode(STranslateContext* pCxt, SDropQnodeStmt* pStmt) { + SDDropQnodeReq dropReq = { .dnodeId = pStmt->dnodeId }; + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_DND_DROP_QNODE; + pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSMCreateDropQSBNodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &dropReq); + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { + SCMCreateTopicReq createReq = {0}; + + if (NULL != pStmt->pQuery) { + pCxt->pParseCxt->topicQuery = true; + int32_t code = translateQuery(pCxt, pStmt->pQuery); + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL); + } + if (TSDB_CODE_SUCCESS != code ) { + return code; + } + } else { + strcpy(createReq.subscribeDbName, pStmt->subscribeDbName); + } + + createReq.sql = strdup(pCxt->pParseCxt->pSql); + if (NULL == createReq.sql) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; + strcpy(name.dbname, pCxt->pParseCxt->db); + strcpy(name.tname, pStmt->topicName); + tNameExtractFullName(&name, createReq.name); + createReq.igExists = pStmt->ignoreExists; + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_TOPIC; + pCxt->pCmdMsg->msgLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSCMCreateTopicReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createReq); + tFreeSCMCreateTopicReq(&createReq); + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt) { + SMDropTopicReq dropReq = {0}; + + SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; + strcpy(name.dbname, pCxt->pParseCxt->db); + strcpy(name.tname, pStmt->topicName); + tNameExtractFullName(&name, dropReq.name); + dropReq.igNotExists = pStmt->ignoreNotExists; + + pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + if (NULL == pCxt->pCmdMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; + pCxt->pCmdMsg->msgType = TDMT_MND_DROP_TOPIC; + pCxt->pCmdMsg->msgLen = tSerializeSMDropTopicReq(NULL, 0, &dropReq); + pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + if (NULL == pCxt->pCmdMsg->pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tSerializeSMDropTopicReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &dropReq); + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { + // todo + return TSDB_CODE_SUCCESS; +} + static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pNode)) { @@ -1221,6 +1543,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_DROP_DATABASE_STMT: code = translateDropDatabase(pCxt, (SDropDatabaseStmt*)pNode); break; + case QUERY_NODE_ALTER_DATABASE_STMT: + code = translateAlterDatabase(pCxt, (SAlterDatabaseStmt*)pNode); + break; case QUERY_NODE_CREATE_TABLE_STMT: code = translateCreateSuperTable(pCxt, (SCreateTableStmt*)pNode); break; @@ -1230,6 +1555,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_DROP_SUPER_TABLE_STMT: code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode); break; + case QUERY_NODE_ALTER_TABLE_STMT: + code = translateAlterTable(pCxt, (SAlterTableStmt*)pNode); + break; case QUERY_NODE_CREATE_USER_STMT: code = translateCreateUser(pCxt, (SCreateUserStmt*)pNode); break; @@ -1248,17 +1576,42 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_DROP_DNODE_STMT: code = translateDropDnode(pCxt, (SDropDnodeStmt*)pNode); break; + case QUERY_NODE_ALTER_DNODE_STMT: + code = translateAlterDnode(pCxt, (SAlterDnodeStmt*)pNode); + break; case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_STABLES_STMT: case QUERY_NODE_SHOW_USERS_STMT: case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: case QUERY_NODE_SHOW_MNODES_STMT: + case QUERY_NODE_SHOW_QNODES_STMT: code = translateShow(pCxt, (SShowStmt*)pNode); break; case QUERY_NODE_SHOW_TABLES_STMT: code = translateShowTables(pCxt); break; + case QUERY_NODE_CREATE_INDEX_STMT: + code = translateCreateIndex(pCxt, (SCreateIndexStmt*)pNode); + break; + case QUERY_NODE_DROP_INDEX_STMT: + code = translateDropIndex(pCxt, (SDropIndexStmt*)pNode); + break; + case QUERY_NODE_CREATE_QNODE_STMT: + code = translateCreateQnode(pCxt, (SCreateQnodeStmt*)pNode); + break; + case QUERY_NODE_DROP_QNODE_STMT: + code = translateDropQnode(pCxt, (SDropQnodeStmt*)pNode); + break; + case QUERY_NODE_CREATE_TOPIC_STMT: + code = translateCreateTopic(pCxt, (SCreateTopicStmt*)pNode); + break; + case QUERY_NODE_DROP_TOPIC_STMT: + code = translateDropTopic(pCxt, (SDropTopicStmt*)pNode); + break; + case QUERY_NODE_ALTER_LOCAL_STMT: + code = translateAlterLocal(pCxt, (SAlterLocalStmt*)pNode); + break; default: break; } @@ -1811,6 +2164,11 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) return rewriteToVnodeModifOpStmt(pQuery, pBufArray); } +static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { + // todo + return TSDB_CODE_SUCCESS; +} + static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pQuery->pRoot)) { @@ -1836,6 +2194,11 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_CREATE_MULTI_TABLE_STMT: code = rewriteCreateMultiTable(pCxt, pQuery); break; + case QUERY_NODE_ALTER_TABLE_STMT: + if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) { + code = rewriteAlterTable(pCxt, pQuery); + } + break; default: break; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index aa2516e2b931557816182e363f73c0f80ab9bec2..69c2380b4538765a68881722f5926d4500f4a28b 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -46,17 +46,19 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_NOT_SINGLE_GROUP: return "Not a single-group group function"; case TSDB_CODE_PAR_TAGS_NOT_MATCHED: - return "tags number not matched"; + return "Tags number not matched"; case TSDB_CODE_PAR_INVALID_TAG_NAME: - return "invalid tag name : %s"; + return "Invalid tag name : %s"; case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG: - return "name or password too long"; + return "Name or password too long"; case TSDB_CODE_PAR_PASSWD_EMPTY: - return "password can not be empty"; + return "Password can not be empty"; case TSDB_CODE_PAR_INVALID_PORT: - return "port should be an integer that is less than 65535 and greater than 0"; + return "Port should be an integer that is less than 65535 and greater than 0"; case TSDB_CODE_PAR_INVALID_ENDPOINT: - return "endpoint should be in the format of 'fqdn:port'"; + return "Endpoint should be in the format of 'fqdn:port'"; + case TSDB_CODE_PAR_EXPRIE_STATEMENT: + return "This statement is no longer supported"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 806e7efc779c27fb1d42173cbd03be30800941ae..8125b807c3765da5ec09749557803bfd528b6d7b 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -98,25 +98,25 @@ # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned char -#define YYNOCODE 218 +#define YYCODETYPE unsigned short int +#define YYNOCODE 256 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SToken yy5; - bool yy25; - SNodeList* yy40; - ENullOrder yy53; - EOrder yy54; - SNode* yy68; - EJoinType yy92; - EFillMode yy94; - SDatabaseOptions* yy339; - SDataType yy372; - EOperatorType yy416; - STableOptions* yy418; + SNodeList* yy24; + int32_t yy68; + ENullOrder yy73; + EJoinType yy84; + EOrder yy130; + EOperatorType yy252; + SAlterOption yy285; + bool yy345; + EFillMode yy358; + SNode* yy360; + SDataType yy400; + SToken yy417; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -131,17 +131,17 @@ typedef union { #define ParseCTX_PARAM #define ParseCTX_FETCH #define ParseCTX_STORE -#define YYNSTATE 290 -#define YYNRULE 248 -#define YYNTOKEN 140 -#define YY_MAX_SHIFT 289 -#define YY_MIN_SHIFTREDUCE 454 -#define YY_MAX_SHIFTREDUCE 701 -#define YY_ERROR_ACTION 702 -#define YY_ACCEPT_ACTION 703 -#define YY_NO_ACTION 704 -#define YY_MIN_REDUCE 705 -#define YY_MAX_REDUCE 952 +#define YYNSTATE 424 +#define YYNRULE 326 +#define YYNTOKEN 163 +#define YY_MAX_SHIFT 423 +#define YY_MIN_SHIFTREDUCE 645 +#define YY_MAX_SHIFTREDUCE 970 +#define YY_ERROR_ACTION 971 +#define YY_ACCEPT_ACTION 972 +#define YY_NO_ACTION 973 +#define YY_MIN_REDUCE 974 +#define YY_MAX_REDUCE 1299 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -208,292 +208,387 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (929) +#define YY_ACTTAB_COUNT (1215) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 41, 155, 23, 96, 793, 200, 791, 845, 200, 884, - /* 10 */ 845, 750, 30, 28, 26, 25, 24, 190, 830, 819, - /* 20 */ 201, 172, 143, 124, 831, 156, 834, 881, 30, 28, - /* 30 */ 26, 25, 24, 137, 716, 821, 819, 177, 821, 819, - /* 40 */ 200, 232, 845, 931, 30, 28, 26, 25, 24, 137, - /* 50 */ 830, 819, 201, 232, 584, 55, 831, 77, 834, 870, - /* 60 */ 744, 929, 159, 869, 866, 793, 10, 791, 574, 282, - /* 70 */ 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, - /* 80 */ 271, 270, 269, 268, 267, 266, 584, 30, 28, 26, - /* 90 */ 25, 24, 220, 608, 200, 220, 845, 22, 147, 573, - /* 100 */ 602, 603, 604, 605, 606, 607, 610, 611, 612, 608, - /* 110 */ 176, 265, 233, 22, 147, 219, 602, 603, 604, 605, - /* 120 */ 606, 607, 610, 611, 612, 232, 78, 754, 258, 668, - /* 130 */ 576, 509, 256, 255, 254, 513, 253, 515, 516, 252, - /* 140 */ 518, 249, 203, 524, 246, 526, 527, 243, 240, 200, - /* 150 */ 10, 845, 931, 168, 666, 667, 669, 670, 190, 830, - /* 160 */ 819, 201, 142, 572, 53, 831, 930, 834, 870, 41, - /* 170 */ 929, 233, 136, 866, 751, 889, 62, 639, 609, 135, - /* 180 */ 749, 565, 20, 229, 931, 132, 754, 563, 884, 186, - /* 190 */ 259, 845, 105, 613, 233, 183, 810, 230, 77, 830, - /* 200 */ 819, 201, 929, 102, 54, 831, 880, 834, 870, 754, - /* 210 */ 61, 94, 145, 866, 72, 193, 217, 216, 215, 286, - /* 220 */ 285, 210, 209, 208, 207, 206, 95, 204, 59, 793, - /* 230 */ 234, 792, 162, 897, 186, 154, 845, 185, 73, 877, - /* 240 */ 878, 56, 882, 78, 830, 819, 201, 756, 68, 54, - /* 250 */ 831, 97, 834, 870, 564, 566, 569, 145, 866, 72, - /* 260 */ 169, 200, 742, 845, 884, 30, 28, 26, 25, 24, - /* 270 */ 188, 830, 819, 201, 78, 577, 54, 831, 898, 834, - /* 280 */ 870, 200, 879, 845, 145, 866, 943, 233, 822, 819, - /* 290 */ 231, 830, 819, 201, 265, 904, 54, 831, 45, 834, - /* 300 */ 870, 200, 754, 845, 145, 866, 943, 233, 643, 747, - /* 310 */ 114, 830, 819, 201, 89, 927, 54, 831, 158, 834, - /* 320 */ 870, 462, 754, 900, 145, 866, 943, 223, 29, 27, - /* 330 */ 756, 68, 29, 27, 644, 888, 599, 565, 757, 68, - /* 340 */ 184, 565, 192, 563, 152, 160, 846, 563, 152, 11, - /* 350 */ 29, 27, 119, 11, 703, 784, 80, 756, 68, 565, - /* 360 */ 233, 29, 27, 161, 2, 563, 152, 665, 19, 1, - /* 370 */ 565, 11, 57, 1, 99, 754, 563, 152, 30, 28, - /* 380 */ 26, 25, 24, 202, 29, 27, 234, 51, 697, 698, - /* 390 */ 234, 1, 222, 565, 901, 63, 86, 462, 746, 563, - /* 400 */ 152, 83, 7, 651, 463, 464, 6, 639, 234, 931, - /* 410 */ 564, 566, 569, 577, 564, 566, 569, 574, 642, 234, - /* 420 */ 178, 173, 171, 77, 911, 1, 183, 929, 197, 78, - /* 430 */ 700, 701, 564, 566, 569, 9, 8, 200, 194, 845, - /* 440 */ 170, 61, 234, 564, 566, 569, 614, 830, 819, 201, - /* 450 */ 190, 31, 55, 831, 167, 834, 870, 9, 8, 59, - /* 460 */ 187, 866, 78, 29, 27, 189, 564, 566, 569, 92, - /* 470 */ 877, 182, 565, 181, 81, 581, 931, 826, 563, 152, - /* 480 */ 31, 200, 824, 845, 569, 165, 29, 27, 620, 910, - /* 490 */ 77, 830, 819, 201, 929, 565, 55, 831, 166, 834, - /* 500 */ 870, 563, 152, 198, 7, 867, 109, 144, 200, 5, - /* 510 */ 845, 195, 79, 85, 26, 25, 24, 108, 830, 819, - /* 520 */ 201, 234, 21, 69, 831, 111, 834, 7, 891, 211, - /* 530 */ 65, 180, 30, 28, 26, 25, 24, 743, 200, 42, - /* 540 */ 845, 88, 106, 164, 234, 564, 566, 569, 830, 819, - /* 550 */ 201, 71, 502, 130, 831, 151, 834, 66, 228, 4, - /* 560 */ 90, 226, 191, 944, 104, 103, 639, 497, 564, 566, - /* 570 */ 569, 200, 57, 845, 50, 60, 576, 47, 885, 91, - /* 580 */ 262, 830, 819, 201, 261, 32, 130, 831, 163, 834, - /* 590 */ 200, 852, 845, 530, 534, 16, 148, 263, 238, 65, - /* 600 */ 830, 819, 201, 946, 196, 126, 831, 539, 834, 200, - /* 610 */ 67, 845, 66, 928, 199, 65, 260, 98, 572, 830, - /* 620 */ 819, 201, 812, 101, 130, 831, 146, 834, 205, 200, - /* 630 */ 212, 845, 214, 40, 213, 218, 221, 578, 179, 830, - /* 640 */ 819, 201, 107, 224, 69, 831, 183, 834, 200, 153, - /* 650 */ 845, 741, 118, 44, 46, 120, 755, 289, 830, 819, - /* 660 */ 201, 61, 236, 125, 831, 200, 834, 845, 115, 133, - /* 670 */ 134, 3, 121, 31, 14, 830, 819, 201, 82, 59, - /* 680 */ 127, 831, 662, 834, 945, 200, 84, 845, 35, 74, - /* 690 */ 877, 878, 664, 882, 262, 830, 819, 201, 261, 70, - /* 700 */ 122, 831, 87, 834, 658, 200, 37, 845, 657, 174, - /* 710 */ 824, 263, 15, 38, 175, 830, 819, 201, 636, 635, - /* 720 */ 128, 831, 18, 834, 200, 93, 845, 117, 33, 34, - /* 730 */ 260, 76, 8, 58, 830, 819, 201, 582, 116, 123, - /* 740 */ 831, 600, 834, 200, 691, 845, 17, 12, 39, 686, - /* 750 */ 685, 149, 690, 830, 819, 201, 689, 100, 129, 831, - /* 760 */ 52, 834, 200, 112, 845, 150, 13, 813, 556, 805, - /* 770 */ 804, 64, 830, 819, 201, 803, 802, 842, 831, 801, - /* 780 */ 834, 200, 800, 845, 799, 798, 558, 797, 796, 795, - /* 790 */ 794, 830, 819, 201, 718, 745, 841, 831, 470, 834, - /* 800 */ 200, 717, 845, 712, 711, 708, 707, 225, 706, 43, - /* 810 */ 830, 819, 201, 227, 47, 840, 831, 110, 834, 200, - /* 820 */ 823, 845, 531, 567, 113, 36, 237, 157, 235, 830, - /* 830 */ 819, 201, 528, 239, 140, 831, 200, 834, 845, 241, - /* 840 */ 244, 247, 508, 250, 242, 523, 830, 819, 201, 522, - /* 850 */ 525, 139, 831, 521, 834, 520, 200, 257, 845, 538, - /* 860 */ 245, 537, 536, 519, 248, 468, 830, 819, 201, 517, - /* 870 */ 251, 141, 831, 48, 834, 49, 200, 264, 845, 489, - /* 880 */ 488, 183, 487, 486, 485, 484, 830, 819, 201, 483, - /* 890 */ 482, 138, 831, 481, 834, 200, 61, 845, 480, 479, - /* 900 */ 478, 477, 476, 475, 474, 830, 819, 201, 473, 710, - /* 910 */ 131, 831, 283, 834, 59, 284, 709, 705, 287, 288, - /* 920 */ 704, 704, 704, 704, 75, 877, 878, 704, 882, + /* 0 */ 1017, 357, 357, 105, 1192, 986, 263, 1075, 1278, 109, + /* 10 */ 42, 341, 1067, 31, 29, 27, 26, 25, 20, 89, + /* 20 */ 1116, 1277, 1078, 1078, 356, 1276, 325, 1073, 31, 29, + /* 30 */ 27, 26, 25, 1063, 314, 77, 912, 356, 76, 75, + /* 40 */ 74, 73, 72, 71, 70, 69, 68, 92, 207, 408, + /* 50 */ 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, + /* 60 */ 397, 396, 395, 394, 393, 392, 391, 390, 975, 24, + /* 70 */ 167, 90, 853, 31, 29, 27, 26, 25, 302, 839, + /* 80 */ 876, 114, 1224, 1225, 1192, 1229, 356, 10, 264, 77, + /* 90 */ 98, 341, 76, 75, 74, 73, 72, 71, 70, 69, + /* 100 */ 68, 244, 31, 29, 27, 26, 25, 285, 207, 280, + /* 110 */ 106, 1045, 284, 1123, 318, 283, 877, 281, 357, 220, + /* 120 */ 282, 416, 415, 354, 1121, 23, 228, 205, 871, 872, + /* 130 */ 873, 874, 875, 879, 880, 881, 10, 1236, 908, 1078, + /* 140 */ 876, 755, 379, 378, 377, 759, 376, 761, 762, 375, + /* 150 */ 764, 372, 423, 770, 369, 772, 773, 366, 363, 104, + /* 160 */ 1177, 31, 29, 27, 26, 25, 183, 1081, 832, 88, + /* 170 */ 911, 239, 57, 30, 28, 412, 877, 182, 1143, 1145, + /* 180 */ 1192, 230, 53, 832, 830, 23, 228, 341, 871, 872, + /* 190 */ 873, 874, 875, 879, 880, 881, 118, 343, 843, 830, + /* 200 */ 277, 1164, 59, 118, 276, 178, 329, 1171, 12, 60, + /* 210 */ 1178, 1181, 1217, 252, 831, 1177, 206, 1213, 344, 1169, + /* 220 */ 190, 233, 234, 131, 1152, 192, 111, 278, 1278, 831, + /* 230 */ 104, 1, 269, 841, 130, 1192, 353, 191, 1080, 357, + /* 240 */ 420, 117, 328, 185, 65, 1276, 1108, 123, 966, 967, + /* 250 */ 305, 273, 343, 146, 313, 420, 1164, 122, 121, 43, + /* 260 */ 1078, 118, 128, 1069, 61, 1178, 1181, 1217, 833, 836, + /* 270 */ 1166, 223, 1213, 112, 238, 1177, 1166, 320, 315, 30, + /* 280 */ 28, 913, 104, 833, 836, 163, 58, 230, 389, 832, + /* 290 */ 1080, 306, 1244, 1177, 221, 1192, 93, 997, 118, 162, + /* 300 */ 236, 1056, 328, 1070, 1164, 830, 127, 63, 325, 878, + /* 310 */ 125, 1164, 343, 1192, 12, 319, 1164, 1164, 21, 42, + /* 320 */ 341, 1177, 1054, 51, 61, 1178, 1181, 1217, 882, 92, + /* 330 */ 343, 223, 1213, 112, 1164, 831, 1074, 1, 1164, 118, + /* 340 */ 1071, 1192, 61, 1178, 1181, 1217, 333, 264, 341, 223, + /* 350 */ 1213, 1290, 1245, 90, 1177, 682, 136, 681, 343, 134, + /* 360 */ 1251, 420, 1164, 115, 1224, 1225, 996, 1229, 389, 1231, + /* 370 */ 61, 1178, 1181, 1217, 1192, 683, 937, 223, 1213, 1290, + /* 380 */ 842, 341, 1177, 1123, 840, 241, 1228, 357, 1274, 833, + /* 390 */ 836, 343, 355, 104, 1144, 1164, 310, 935, 936, 938, + /* 400 */ 939, 1080, 1192, 61, 1178, 1181, 1217, 1164, 1078, 341, + /* 410 */ 223, 1213, 1290, 299, 9, 8, 30, 28, 334, 343, + /* 420 */ 1177, 1235, 382, 1164, 230, 1055, 832, 1231, 329, 30, + /* 430 */ 28, 195, 1178, 1181, 1053, 1123, 995, 230, 357, 832, + /* 440 */ 1192, 235, 830, 180, 1227, 325, 1121, 341, 844, 1065, + /* 450 */ 1278, 12, 920, 300, 1123, 830, 1177, 343, 841, 1078, + /* 460 */ 240, 1164, 1061, 117, 229, 1121, 92, 1276, 357, 201, + /* 470 */ 1178, 1181, 831, 65, 1, 1278, 1192, 1164, 994, 993, + /* 480 */ 279, 381, 386, 341, 889, 831, 385, 7, 117, 1078, + /* 490 */ 90, 386, 1276, 343, 1177, 385, 987, 1164, 420, 327, + /* 500 */ 113, 1224, 1225, 1140, 1229, 62, 1178, 1181, 1217, 387, + /* 510 */ 120, 420, 1216, 1213, 1192, 325, 1231, 311, 387, 1164, + /* 520 */ 1164, 341, 27, 26, 25, 1046, 833, 836, 384, 383, + /* 530 */ 1123, 343, 992, 1226, 991, 1164, 92, 384, 383, 833, + /* 540 */ 836, 1122, 344, 62, 1178, 1181, 1217, 357, 1153, 149, + /* 550 */ 339, 1213, 242, 164, 118, 329, 30, 28, 342, 990, + /* 560 */ 90, 9, 8, 989, 230, 988, 832, 1167, 1078, 985, + /* 570 */ 160, 1224, 324, 1164, 323, 1164, 1177, 1278, 984, 30, + /* 580 */ 28, 681, 830, 30, 28, 332, 1117, 230, 983, 832, + /* 590 */ 117, 230, 839, 832, 1276, 1177, 1192, 271, 1013, 245, + /* 600 */ 1164, 982, 257, 341, 1164, 830, 1164, 981, 1164, 830, + /* 610 */ 1164, 258, 831, 343, 7, 1192, 934, 1164, 980, 1164, + /* 620 */ 286, 6, 341, 979, 157, 107, 1178, 1181, 78, 1164, + /* 630 */ 138, 214, 343, 137, 1008, 831, 1164, 7, 420, 831, + /* 640 */ 140, 1, 1164, 139, 62, 1178, 1181, 1217, 1164, 340, + /* 650 */ 1177, 978, 1214, 277, 1006, 977, 288, 276, 336, 1164, + /* 660 */ 297, 420, 330, 1291, 1164, 420, 833, 836, 972, 215, + /* 670 */ 1192, 213, 212, 295, 275, 1177, 291, 341, 256, 974, + /* 680 */ 278, 251, 250, 249, 248, 247, 154, 343, 270, 833, + /* 690 */ 836, 1164, 1164, 833, 836, 1192, 1164, 1247, 152, 200, + /* 700 */ 1178, 1181, 341, 87, 86, 85, 84, 83, 82, 81, + /* 710 */ 80, 79, 343, 1177, 908, 142, 1164, 243, 141, 307, + /* 720 */ 326, 1193, 969, 970, 201, 1178, 1181, 166, 868, 883, + /* 730 */ 321, 1177, 337, 1192, 2, 331, 850, 825, 839, 1278, + /* 740 */ 341, 32, 1142, 172, 119, 253, 1177, 349, 32, 32, + /* 750 */ 343, 1192, 117, 177, 1164, 170, 1276, 1177, 341, 95, + /* 760 */ 254, 255, 107, 1178, 1181, 96, 1192, 748, 343, 246, + /* 770 */ 847, 259, 1164, 341, 743, 227, 776, 1192, 124, 98, + /* 780 */ 201, 1178, 1181, 343, 341, 260, 78, 1164, 361, 1177, + /* 790 */ 231, 780, 1177, 261, 343, 201, 1178, 1181, 1164, 846, + /* 800 */ 1292, 786, 785, 96, 265, 262, 199, 1178, 1181, 1192, + /* 810 */ 22, 41, 1192, 97, 98, 99, 341, 1177, 129, 341, + /* 820 */ 31, 29, 27, 26, 25, 845, 343, 96, 1177, 343, + /* 830 */ 1164, 272, 274, 1164, 1068, 133, 1064, 1192, 202, 1178, + /* 840 */ 1181, 193, 1178, 1181, 341, 135, 100, 101, 1192, 67, + /* 850 */ 1066, 219, 145, 1062, 343, 341, 1177, 102, 1164, 103, + /* 860 */ 304, 1177, 303, 301, 844, 343, 203, 1178, 1181, 1164, + /* 870 */ 312, 1248, 1177, 1258, 836, 309, 1192, 194, 1178, 1181, + /* 880 */ 347, 1192, 150, 341, 1257, 5, 153, 222, 341, 322, + /* 890 */ 908, 91, 1192, 343, 1238, 156, 308, 1164, 343, 341, + /* 900 */ 1177, 4, 1164, 843, 1232, 204, 1178, 1181, 110, 343, + /* 910 */ 1189, 1178, 1181, 1164, 33, 224, 338, 159, 1177, 335, + /* 920 */ 1192, 1188, 1178, 1181, 158, 17, 345, 341, 1199, 1151, + /* 930 */ 1293, 346, 350, 1177, 1275, 165, 1150, 343, 1192, 232, + /* 940 */ 351, 1164, 174, 352, 1177, 341, 184, 50, 52, 1187, + /* 950 */ 1178, 1181, 1079, 1192, 186, 343, 181, 359, 419, 1164, + /* 960 */ 341, 196, 188, 197, 1192, 189, 1158, 210, 1178, 1181, + /* 970 */ 343, 341, 290, 808, 1164, 1135, 1177, 1134, 94, 1177, + /* 980 */ 1133, 343, 209, 1178, 1181, 1164, 1132, 298, 1131, 1130, + /* 990 */ 1129, 1128, 810, 211, 1178, 1181, 1192, 1020, 1127, 1192, + /* 1000 */ 1126, 144, 1125, 341, 293, 1124, 341, 1019, 1157, 287, + /* 1010 */ 1148, 1057, 143, 343, 126, 694, 343, 1164, 1018, 1016, + /* 1020 */ 1164, 266, 268, 267, 1005, 208, 1178, 1181, 198, 1178, + /* 1030 */ 1181, 1004, 1001, 1059, 66, 132, 789, 38, 1058, 791, + /* 1040 */ 37, 31, 29, 27, 26, 25, 285, 790, 280, 1014, + /* 1050 */ 723, 284, 216, 722, 283, 721, 281, 1009, 720, 282, + /* 1060 */ 719, 718, 217, 1007, 289, 218, 292, 1000, 294, 999, + /* 1070 */ 296, 64, 1156, 1155, 36, 1147, 44, 147, 148, 3, + /* 1080 */ 14, 15, 32, 316, 151, 34, 39, 933, 11, 47, + /* 1090 */ 8, 108, 348, 1146, 955, 869, 155, 954, 927, 225, + /* 1100 */ 853, 176, 959, 958, 226, 45, 973, 926, 973, 973, + /* 1110 */ 973, 46, 973, 175, 973, 905, 973, 973, 904, 973, + /* 1120 */ 317, 1169, 973, 973, 960, 19, 973, 360, 237, 973, + /* 1130 */ 364, 851, 161, 169, 754, 35, 116, 367, 16, 931, + /* 1140 */ 168, 13, 370, 373, 18, 973, 171, 173, 48, 49, + /* 1150 */ 782, 1015, 1003, 40, 1002, 714, 706, 777, 774, 784, + /* 1160 */ 53, 362, 973, 1168, 179, 358, 365, 783, 771, 765, + /* 1170 */ 368, 371, 388, 763, 692, 713, 374, 712, 711, 710, + /* 1180 */ 709, 708, 707, 705, 410, 704, 54, 703, 702, 55, + /* 1190 */ 769, 768, 56, 998, 701, 700, 699, 698, 380, 767, + /* 1200 */ 715, 417, 697, 409, 766, 411, 413, 414, 418, 973, + /* 1210 */ 834, 187, 421, 973, 422, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 147, 159, 180, 181, 162, 160, 164, 162, 160, 173, - /* 10 */ 162, 158, 12, 13, 14, 15, 16, 169, 170, 171, - /* 20 */ 172, 176, 157, 175, 176, 157, 178, 191, 12, 13, - /* 30 */ 14, 15, 16, 33, 0, 170, 171, 28, 170, 171, - /* 40 */ 160, 28, 162, 195, 12, 13, 14, 15, 16, 33, - /* 50 */ 170, 171, 172, 28, 54, 175, 176, 209, 178, 179, - /* 60 */ 0, 213, 159, 183, 184, 162, 53, 164, 28, 35, - /* 70 */ 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, - /* 80 */ 46, 47, 48, 49, 50, 51, 54, 12, 13, 14, - /* 90 */ 15, 16, 32, 93, 160, 32, 162, 97, 98, 28, - /* 100 */ 100, 101, 102, 103, 104, 105, 106, 107, 108, 93, - /* 110 */ 176, 32, 145, 97, 98, 148, 100, 101, 102, 103, - /* 120 */ 104, 105, 106, 107, 108, 28, 113, 160, 163, 99, - /* 130 */ 28, 61, 62, 63, 64, 65, 66, 67, 68, 69, - /* 140 */ 70, 71, 145, 73, 74, 75, 76, 77, 78, 160, - /* 150 */ 53, 162, 195, 123, 124, 125, 126, 127, 169, 170, - /* 160 */ 171, 172, 165, 28, 175, 176, 209, 178, 179, 147, - /* 170 */ 213, 145, 183, 184, 148, 110, 154, 112, 93, 18, - /* 180 */ 158, 21, 97, 22, 195, 24, 160, 27, 173, 160, - /* 190 */ 57, 162, 31, 108, 145, 145, 160, 148, 209, 170, - /* 200 */ 171, 172, 213, 167, 175, 176, 191, 178, 179, 160, - /* 210 */ 160, 109, 183, 184, 185, 3, 81, 82, 83, 142, - /* 220 */ 143, 86, 87, 88, 89, 90, 197, 92, 178, 162, - /* 230 */ 70, 164, 203, 204, 160, 149, 162, 187, 188, 189, - /* 240 */ 190, 80, 192, 113, 170, 171, 172, 161, 162, 175, - /* 250 */ 176, 216, 178, 179, 94, 95, 96, 183, 184, 185, - /* 260 */ 207, 160, 0, 162, 173, 12, 13, 14, 15, 16, - /* 270 */ 33, 170, 171, 172, 113, 28, 175, 176, 204, 178, - /* 280 */ 179, 160, 191, 162, 183, 184, 185, 145, 170, 171, - /* 290 */ 148, 170, 171, 172, 32, 194, 175, 176, 144, 178, - /* 300 */ 179, 160, 160, 162, 183, 184, 185, 145, 4, 155, - /* 310 */ 148, 170, 171, 172, 200, 194, 175, 176, 149, 178, - /* 320 */ 179, 21, 160, 174, 183, 184, 185, 27, 12, 13, - /* 330 */ 161, 162, 12, 13, 14, 194, 99, 21, 161, 162, - /* 340 */ 193, 21, 130, 27, 28, 149, 162, 27, 28, 33, - /* 350 */ 12, 13, 150, 33, 140, 153, 109, 161, 162, 21, - /* 360 */ 145, 12, 13, 148, 196, 27, 28, 54, 2, 53, - /* 370 */ 21, 33, 59, 53, 210, 160, 27, 28, 12, 13, - /* 380 */ 14, 15, 16, 169, 12, 13, 70, 144, 135, 136, - /* 390 */ 70, 53, 142, 21, 174, 152, 54, 21, 155, 27, - /* 400 */ 28, 59, 53, 14, 28, 29, 111, 112, 70, 195, - /* 410 */ 94, 95, 96, 28, 94, 95, 96, 28, 114, 70, - /* 420 */ 118, 119, 120, 209, 206, 53, 145, 213, 59, 113, - /* 430 */ 138, 139, 94, 95, 96, 1, 2, 160, 59, 162, - /* 440 */ 122, 160, 70, 94, 95, 96, 54, 170, 171, 172, - /* 450 */ 169, 59, 175, 176, 121, 178, 179, 1, 2, 178, - /* 460 */ 183, 184, 113, 12, 13, 14, 94, 95, 96, 188, - /* 470 */ 189, 190, 21, 192, 205, 54, 195, 53, 27, 28, - /* 480 */ 59, 160, 58, 162, 96, 171, 12, 13, 54, 206, - /* 490 */ 209, 170, 171, 172, 213, 21, 175, 176, 171, 178, - /* 500 */ 179, 27, 28, 134, 53, 184, 19, 171, 160, 129, - /* 510 */ 162, 132, 25, 205, 14, 15, 16, 30, 170, 171, - /* 520 */ 172, 70, 2, 175, 176, 54, 178, 53, 202, 49, - /* 530 */ 59, 128, 12, 13, 14, 15, 16, 0, 160, 52, - /* 540 */ 162, 201, 55, 116, 70, 94, 95, 96, 170, 171, - /* 550 */ 172, 199, 54, 175, 176, 177, 178, 59, 20, 115, - /* 560 */ 198, 23, 214, 215, 84, 85, 112, 54, 94, 95, - /* 570 */ 96, 160, 59, 162, 53, 160, 28, 56, 173, 186, - /* 580 */ 43, 170, 171, 172, 47, 91, 175, 176, 177, 178, - /* 590 */ 160, 182, 162, 54, 54, 53, 137, 60, 59, 59, - /* 600 */ 170, 171, 172, 217, 131, 175, 176, 54, 178, 160, - /* 610 */ 54, 162, 59, 212, 133, 59, 79, 211, 28, 170, - /* 620 */ 171, 172, 145, 91, 175, 176, 177, 178, 168, 160, - /* 630 */ 166, 162, 166, 147, 93, 145, 145, 28, 208, 170, - /* 640 */ 171, 172, 147, 141, 175, 176, 145, 178, 160, 141, - /* 650 */ 162, 0, 153, 144, 53, 145, 160, 141, 170, 171, - /* 660 */ 172, 160, 156, 175, 176, 160, 178, 162, 144, 151, - /* 670 */ 151, 59, 146, 59, 117, 170, 171, 172, 54, 178, - /* 680 */ 175, 176, 54, 178, 215, 160, 53, 162, 59, 188, - /* 690 */ 189, 190, 54, 192, 43, 170, 171, 172, 47, 53, - /* 700 */ 175, 176, 53, 178, 54, 160, 53, 162, 54, 27, - /* 710 */ 58, 60, 117, 53, 59, 170, 171, 172, 54, 54, - /* 720 */ 175, 176, 59, 178, 160, 58, 162, 19, 110, 59, - /* 730 */ 79, 58, 2, 25, 170, 171, 172, 54, 30, 175, - /* 740 */ 176, 99, 178, 160, 54, 162, 59, 117, 4, 27, - /* 750 */ 27, 27, 27, 170, 171, 172, 27, 58, 175, 176, - /* 760 */ 52, 178, 160, 55, 162, 27, 53, 0, 58, 0, - /* 770 */ 0, 91, 170, 171, 172, 0, 0, 175, 176, 0, - /* 780 */ 178, 160, 0, 162, 0, 0, 21, 0, 0, 0, - /* 790 */ 0, 170, 171, 172, 0, 0, 175, 176, 34, 178, - /* 800 */ 160, 0, 162, 0, 0, 0, 0, 21, 0, 53, - /* 810 */ 170, 171, 172, 21, 56, 175, 176, 19, 178, 160, - /* 820 */ 58, 162, 54, 21, 58, 53, 27, 27, 57, 170, - /* 830 */ 171, 172, 54, 53, 175, 176, 160, 178, 162, 27, - /* 840 */ 27, 27, 21, 27, 53, 72, 170, 171, 172, 72, - /* 850 */ 54, 175, 176, 72, 178, 72, 160, 60, 162, 27, - /* 860 */ 53, 27, 21, 54, 53, 34, 170, 171, 172, 54, - /* 870 */ 53, 175, 176, 53, 178, 53, 160, 33, 162, 27, - /* 880 */ 27, 145, 27, 27, 27, 27, 170, 171, 172, 27, - /* 890 */ 21, 175, 176, 27, 178, 160, 160, 162, 27, 27, - /* 900 */ 27, 27, 27, 27, 27, 170, 171, 172, 27, 0, - /* 910 */ 175, 176, 27, 178, 178, 26, 0, 0, 21, 20, - /* 920 */ 218, 218, 218, 218, 188, 189, 190, 218, 192, 218, - /* 930 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 940 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 950 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 960 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 970 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 980 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 990 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 1000 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 1010 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 1020 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 1030 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 1040 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, - /* 1050 */ 218, 218, 218, 218, 218, 218, 218, 218, 218, 218, + /* 0 */ 0, 172, 172, 165, 186, 167, 177, 177, 234, 185, + /* 10 */ 174, 193, 187, 12, 13, 14, 15, 16, 2, 183, + /* 20 */ 196, 247, 193, 193, 20, 251, 172, 191, 12, 13, + /* 30 */ 14, 15, 16, 187, 216, 21, 4, 20, 24, 25, + /* 40 */ 26, 27, 28, 29, 30, 31, 32, 193, 47, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + /* 60 */ 60, 61, 62, 63, 64, 65, 66, 67, 0, 219, + /* 70 */ 220, 217, 71, 12, 13, 14, 15, 16, 71, 20, + /* 80 */ 79, 227, 228, 229, 186, 231, 20, 70, 46, 21, + /* 90 */ 83, 193, 24, 25, 26, 27, 28, 29, 30, 31, + /* 100 */ 32, 172, 12, 13, 14, 15, 16, 49, 47, 51, + /* 110 */ 175, 176, 54, 186, 216, 57, 115, 59, 172, 192, + /* 120 */ 62, 169, 170, 177, 197, 124, 125, 198, 127, 128, + /* 130 */ 129, 130, 131, 132, 133, 134, 70, 135, 136, 193, + /* 140 */ 79, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 150 */ 94, 95, 19, 97, 98, 99, 100, 101, 102, 186, + /* 160 */ 166, 12, 13, 14, 15, 16, 33, 194, 22, 36, + /* 170 */ 138, 195, 70, 12, 13, 42, 115, 44, 202, 203, + /* 180 */ 186, 20, 80, 22, 38, 124, 125, 193, 127, 128, + /* 190 */ 129, 130, 131, 132, 133, 134, 137, 203, 20, 38, + /* 200 */ 57, 207, 69, 137, 61, 72, 212, 70, 47, 215, + /* 210 */ 216, 217, 218, 63, 68, 166, 222, 223, 203, 82, + /* 220 */ 18, 206, 178, 33, 209, 23, 36, 84, 234, 68, + /* 230 */ 186, 70, 42, 20, 44, 186, 103, 35, 194, 172, + /* 240 */ 94, 247, 193, 179, 177, 251, 182, 45, 158, 159, + /* 250 */ 117, 184, 203, 120, 119, 94, 207, 107, 108, 69, + /* 260 */ 193, 137, 72, 166, 215, 216, 217, 218, 122, 123, + /* 270 */ 166, 222, 223, 224, 178, 166, 166, 142, 143, 12, + /* 280 */ 13, 14, 186, 122, 123, 236, 171, 20, 46, 22, + /* 290 */ 194, 242, 243, 166, 190, 186, 181, 166, 137, 121, + /* 300 */ 190, 0, 193, 188, 207, 38, 116, 105, 172, 115, + /* 310 */ 120, 207, 203, 186, 47, 20, 207, 207, 124, 174, + /* 320 */ 193, 166, 0, 171, 215, 216, 217, 218, 134, 193, + /* 330 */ 203, 222, 223, 224, 207, 68, 191, 70, 207, 137, + /* 340 */ 188, 186, 215, 216, 217, 218, 83, 46, 193, 222, + /* 350 */ 223, 224, 243, 217, 166, 20, 74, 22, 203, 77, + /* 360 */ 233, 94, 207, 227, 228, 229, 166, 231, 46, 213, + /* 370 */ 215, 216, 217, 218, 186, 40, 126, 222, 223, 224, + /* 380 */ 20, 193, 166, 186, 20, 178, 230, 172, 233, 122, + /* 390 */ 123, 203, 177, 186, 197, 207, 146, 147, 148, 149, + /* 400 */ 150, 194, 186, 215, 216, 217, 218, 207, 193, 193, + /* 410 */ 222, 223, 224, 172, 1, 2, 12, 13, 155, 203, + /* 420 */ 166, 233, 81, 207, 20, 0, 22, 213, 212, 12, + /* 430 */ 13, 215, 216, 217, 0, 186, 166, 20, 172, 22, + /* 440 */ 186, 192, 38, 177, 230, 172, 197, 193, 20, 187, + /* 450 */ 234, 47, 14, 212, 186, 38, 166, 203, 20, 193, + /* 460 */ 192, 207, 187, 247, 210, 197, 193, 251, 172, 215, + /* 470 */ 216, 217, 68, 177, 70, 234, 186, 207, 166, 166, + /* 480 */ 184, 187, 57, 193, 71, 68, 61, 70, 247, 193, + /* 490 */ 217, 57, 251, 203, 166, 61, 167, 207, 94, 226, + /* 500 */ 227, 228, 229, 193, 231, 215, 216, 217, 218, 84, + /* 510 */ 200, 94, 222, 223, 186, 172, 213, 245, 84, 207, + /* 520 */ 207, 193, 14, 15, 16, 176, 122, 123, 103, 104, + /* 530 */ 186, 203, 166, 230, 166, 207, 193, 103, 104, 122, + /* 540 */ 123, 197, 203, 215, 216, 217, 218, 172, 209, 121, + /* 550 */ 222, 223, 177, 254, 137, 212, 12, 13, 14, 166, + /* 560 */ 217, 1, 2, 166, 20, 166, 22, 166, 193, 166, + /* 570 */ 227, 228, 229, 207, 231, 207, 166, 234, 166, 12, + /* 580 */ 13, 22, 38, 12, 13, 3, 196, 20, 166, 22, + /* 590 */ 247, 20, 20, 22, 251, 166, 186, 38, 0, 27, + /* 600 */ 207, 166, 30, 193, 207, 38, 207, 166, 207, 38, + /* 610 */ 207, 39, 68, 203, 70, 186, 71, 207, 166, 207, + /* 620 */ 22, 43, 193, 166, 239, 215, 216, 217, 83, 207, + /* 630 */ 74, 35, 203, 77, 0, 68, 207, 70, 94, 68, + /* 640 */ 74, 70, 207, 77, 215, 216, 217, 218, 207, 47, + /* 650 */ 166, 166, 223, 57, 0, 166, 22, 61, 83, 207, + /* 660 */ 21, 94, 252, 253, 207, 94, 122, 123, 163, 73, + /* 670 */ 186, 75, 76, 34, 78, 166, 22, 193, 106, 0, + /* 680 */ 84, 109, 110, 111, 112, 113, 71, 203, 169, 122, + /* 690 */ 123, 207, 207, 122, 123, 186, 207, 214, 83, 215, + /* 700 */ 216, 217, 193, 24, 25, 26, 27, 28, 29, 30, + /* 710 */ 31, 32, 203, 166, 136, 74, 207, 212, 77, 210, + /* 720 */ 232, 186, 161, 162, 215, 216, 217, 248, 126, 71, + /* 730 */ 246, 166, 157, 186, 235, 153, 71, 71, 20, 234, + /* 740 */ 193, 83, 172, 71, 114, 199, 166, 71, 83, 83, + /* 750 */ 203, 186, 247, 71, 207, 83, 251, 166, 193, 83, + /* 760 */ 115, 199, 215, 216, 217, 83, 186, 71, 203, 201, + /* 770 */ 20, 172, 207, 193, 71, 210, 71, 186, 174, 83, + /* 780 */ 215, 216, 217, 203, 193, 211, 83, 207, 83, 166, + /* 790 */ 210, 71, 166, 193, 203, 215, 216, 217, 207, 20, + /* 800 */ 253, 71, 71, 83, 172, 204, 215, 216, 217, 186, + /* 810 */ 2, 174, 186, 83, 83, 71, 193, 166, 174, 193, + /* 820 */ 12, 13, 14, 15, 16, 20, 203, 83, 166, 203, + /* 830 */ 207, 168, 186, 207, 186, 186, 186, 186, 215, 216, + /* 840 */ 217, 215, 216, 217, 193, 186, 186, 186, 186, 172, + /* 850 */ 186, 168, 171, 186, 203, 193, 166, 186, 207, 186, + /* 860 */ 204, 166, 193, 211, 20, 203, 215, 216, 217, 207, + /* 870 */ 145, 214, 166, 244, 123, 207, 186, 215, 216, 217, + /* 880 */ 144, 186, 208, 193, 244, 152, 208, 207, 193, 151, + /* 890 */ 136, 193, 186, 203, 241, 240, 140, 207, 203, 193, + /* 900 */ 166, 139, 207, 20, 213, 215, 216, 217, 238, 203, + /* 910 */ 215, 216, 217, 207, 114, 160, 156, 225, 166, 154, + /* 920 */ 186, 215, 216, 217, 237, 70, 207, 193, 221, 208, + /* 930 */ 255, 207, 118, 166, 250, 249, 208, 203, 186, 207, + /* 940 */ 205, 207, 193, 204, 166, 193, 182, 171, 70, 215, + /* 950 */ 216, 217, 193, 186, 172, 203, 171, 189, 168, 207, + /* 960 */ 193, 180, 173, 180, 186, 164, 0, 215, 216, 217, + /* 970 */ 203, 193, 4, 82, 207, 0, 166, 0, 114, 166, + /* 980 */ 0, 203, 215, 216, 217, 207, 0, 19, 0, 0, + /* 990 */ 0, 0, 22, 215, 216, 217, 186, 0, 0, 186, + /* 1000 */ 0, 33, 0, 193, 36, 0, 193, 0, 0, 41, + /* 1010 */ 0, 0, 44, 203, 43, 48, 203, 207, 0, 0, + /* 1020 */ 207, 38, 43, 36, 0, 215, 216, 217, 215, 216, + /* 1030 */ 217, 0, 0, 0, 79, 77, 22, 69, 0, 38, + /* 1040 */ 72, 12, 13, 14, 15, 16, 49, 38, 51, 0, + /* 1050 */ 38, 54, 22, 38, 57, 38, 59, 0, 38, 62, + /* 1060 */ 38, 38, 22, 0, 39, 22, 38, 0, 22, 0, + /* 1070 */ 22, 20, 0, 0, 121, 0, 70, 43, 116, 83, + /* 1080 */ 141, 141, 83, 38, 71, 135, 83, 71, 141, 4, + /* 1090 */ 2, 70, 119, 0, 38, 126, 70, 38, 71, 38, + /* 1100 */ 71, 116, 38, 38, 38, 70, 256, 71, 256, 256, + /* 1110 */ 256, 70, 256, 43, 256, 71, 256, 256, 71, 256, + /* 1120 */ 83, 82, 256, 256, 71, 83, 256, 38, 38, 256, + /* 1130 */ 38, 71, 82, 71, 22, 83, 82, 38, 83, 71, + /* 1140 */ 82, 70, 38, 38, 70, 256, 70, 70, 70, 70, + /* 1150 */ 22, 0, 0, 70, 0, 22, 22, 71, 71, 38, + /* 1160 */ 80, 70, 256, 82, 82, 81, 70, 38, 71, 71, + /* 1170 */ 70, 70, 47, 71, 48, 38, 70, 38, 38, 38, + /* 1180 */ 38, 38, 38, 38, 36, 38, 70, 38, 38, 70, + /* 1190 */ 96, 96, 70, 0, 38, 38, 38, 38, 84, 96, + /* 1200 */ 68, 22, 38, 38, 96, 43, 38, 37, 21, 256, + /* 1210 */ 22, 22, 21, 256, 20, 256, 256, 256, 256, 256, + /* 1220 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1230 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1240 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1250 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1260 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1270 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1280 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1290 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1300 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1310 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1320 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1330 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1340 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1350 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1360 */ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + /* 1370 */ 256, 256, 256, 256, 256, 256, 256, 256, }; -#define YY_SHIFT_COUNT (289) +#define YY_SHIFT_COUNT (423) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (917) +#define YY_SHIFT_MAX (1194) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 161, 316, 320, 338, 338, 338, 338, 349, 338, 338, - /* 10 */ 13, 372, 474, 451, 474, 474, 474, 474, 474, 474, - /* 20 */ 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, - /* 30 */ 474, 474, 97, 97, 97, 160, 160, 9, 9, 130, - /* 40 */ 25, 25, 63, 40, 25, 25, 40, 25, 40, 40, - /* 50 */ 40, 25, 79, 0, 16, 16, 135, 160, 376, 102, - /* 60 */ 102, 102, 60, 262, 71, 40, 40, 133, 70, 253, - /* 70 */ 30, 302, 247, 65, 295, 65, 389, 212, 304, 300, - /* 80 */ 385, 318, 333, 388, 388, 318, 333, 388, 380, 403, - /* 90 */ 427, 444, 454, 71, 548, 494, 542, 459, 481, 473, - /* 100 */ 40, 590, 532, 541, 541, 590, 63, 590, 63, 609, - /* 110 */ 609, 133, 79, 71, 601, 590, 79, 609, 929, 929, - /* 120 */ 929, 34, 366, 520, 32, 75, 75, 75, 75, 75, - /* 130 */ 75, 75, 487, 537, 651, 708, 434, 85, 500, 500, - /* 140 */ 500, 500, 480, 313, 342, 456, 392, 237, 292, 379, - /* 150 */ 369, 421, 424, 538, 471, 498, 513, 539, 540, 553, - /* 160 */ 556, 521, 612, 614, 557, 624, 628, 633, 629, 638, - /* 170 */ 646, 649, 650, 653, 654, 682, 655, 652, 660, 663, - /* 180 */ 595, 664, 665, 667, 618, 670, 673, 730, 642, 683, - /* 190 */ 690, 687, 630, 744, 722, 723, 724, 725, 729, 738, - /* 200 */ 699, 713, 767, 710, 769, 770, 680, 775, 776, 779, - /* 210 */ 782, 784, 785, 765, 787, 788, 789, 790, 794, 795, - /* 220 */ 764, 801, 803, 804, 805, 806, 786, 808, 792, 798, - /* 230 */ 756, 758, 762, 766, 802, 772, 771, 768, 799, 800, - /* 240 */ 780, 778, 812, 791, 796, 813, 807, 809, 814, 811, - /* 250 */ 815, 816, 817, 773, 777, 781, 783, 821, 797, 820, - /* 260 */ 822, 832, 834, 841, 831, 844, 852, 853, 855, 856, - /* 270 */ 857, 858, 862, 869, 866, 871, 872, 873, 874, 875, - /* 280 */ 876, 877, 881, 909, 885, 889, 916, 917, 897, 899, + /* 0 */ 202, 161, 267, 404, 404, 404, 404, 417, 404, 404, + /* 10 */ 66, 567, 571, 544, 567, 567, 567, 567, 567, 567, + /* 20 */ 567, 567, 567, 567, 567, 567, 567, 567, 567, 567, + /* 30 */ 567, 567, 567, 17, 17, 17, 59, 4, 4, 146, + /* 40 */ 146, 4, 4, 42, 213, 295, 295, 124, 360, 213, + /* 50 */ 4, 4, 213, 4, 213, 360, 213, 213, 4, 242, + /* 60 */ 1, 61, 61, 572, 14, 596, 146, 58, 146, 146, + /* 70 */ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + /* 80 */ 146, 146, 146, 146, 146, 146, 146, 146, 335, 301, + /* 90 */ 178, 178, 178, 322, 364, 360, 213, 213, 213, 341, + /* 100 */ 56, 56, 56, 56, 56, 68, 997, 90, 250, 143, + /* 110 */ 135, 559, 428, 2, 578, 2, 438, 582, 32, 718, + /* 120 */ 630, 645, 645, 718, 750, 42, 364, 779, 42, 718, + /* 130 */ 42, 805, 213, 213, 213, 213, 213, 213, 213, 213, + /* 140 */ 213, 213, 213, 718, 805, 750, 242, 364, 779, 844, + /* 150 */ 725, 736, 751, 725, 736, 751, 733, 738, 756, 762, + /* 160 */ 754, 364, 883, 800, 755, 760, 765, 855, 213, 736, + /* 170 */ 751, 751, 736, 751, 814, 364, 779, 341, 242, 364, + /* 180 */ 878, 718, 242, 805, 1215, 1215, 1215, 1215, 0, 679, + /* 190 */ 133, 190, 968, 16, 808, 1029, 425, 434, 149, 149, + /* 200 */ 149, 149, 149, 149, 149, 150, 413, 194, 508, 508, + /* 210 */ 508, 508, 282, 556, 566, 641, 598, 634, 654, 639, + /* 220 */ 7, 545, 615, 560, 561, 263, 575, 658, 602, 665, + /* 230 */ 137, 666, 672, 676, 682, 696, 703, 705, 720, 730, + /* 240 */ 731, 744, 102, 966, 891, 975, 977, 864, 980, 986, + /* 250 */ 988, 989, 990, 991, 970, 998, 1000, 1002, 1005, 1007, + /* 260 */ 1008, 1010, 971, 1011, 967, 1018, 1019, 983, 987, 979, + /* 270 */ 1024, 1031, 1032, 1033, 955, 958, 1001, 1009, 1014, 1038, + /* 280 */ 1012, 1015, 1017, 1020, 1022, 1023, 1049, 1030, 1057, 1040, + /* 290 */ 1025, 1063, 1043, 1028, 1067, 1046, 1069, 1048, 1051, 1072, + /* 300 */ 1073, 953, 1075, 1006, 1034, 962, 996, 999, 939, 1013, + /* 310 */ 1003, 1016, 1021, 1026, 1027, 1035, 1036, 1045, 1037, 1039, + /* 320 */ 1041, 1042, 940, 1044, 1047, 1050, 950, 1052, 1054, 1053, + /* 330 */ 1055, 947, 1085, 1056, 1059, 1061, 1064, 1065, 1066, 1088, + /* 340 */ 969, 1058, 1060, 1071, 1074, 1062, 1068, 1076, 1077, 973, + /* 350 */ 1078, 1093, 1070, 985, 1079, 1080, 1081, 1082, 1083, 1084, + /* 360 */ 1086, 1089, 1090, 1091, 1087, 1092, 1096, 1097, 1099, 1100, + /* 370 */ 1098, 1104, 1101, 1102, 1105, 1106, 1094, 1095, 1103, 1108, + /* 380 */ 1112, 1114, 1116, 1119, 1122, 1121, 1129, 1128, 1126, 1125, + /* 390 */ 1132, 1133, 1137, 1139, 1140, 1141, 1142, 1143, 1144, 1134, + /* 400 */ 1145, 1147, 1149, 1150, 1156, 1157, 1158, 1159, 1164, 1151, + /* 410 */ 1165, 1148, 1162, 1152, 1168, 1170, 1154, 1193, 1179, 1187, + /* 420 */ 1188, 1189, 1191, 1194, }; -#define YY_REDUCE_COUNT (120) -#define YY_REDUCE_MIN (-178) -#define YY_REDUCE_MAX (736) +#define YY_REDUCE_COUNT (187) +#define YY_REDUCE_MIN (-226) +#define YY_REDUCE_MAX (813) static const short yy_reduce_ofst[] = { - /* 0 */ 214, -11, 29, 74, 101, 121, 141, -152, -120, 277, - /* 10 */ 281, 321, 348, 378, 411, 430, 449, 469, 488, 505, - /* 20 */ 525, 545, 564, 583, 602, 621, 640, 659, 676, 696, - /* 30 */ 716, 735, 50, 501, 736, -135, -132, -155, -66, -43, - /* 40 */ -33, 26, 22, 86, 49, 142, -158, 162, 169, -97, - /* 50 */ 196, 215, 243, -178, -178, -178, -3, 118, 77, -164, - /* 60 */ 15, 91, -147, 154, 36, 177, 67, 202, -35, 35, - /* 70 */ 53, 114, 149, 147, 147, 147, 184, 164, 168, 250, - /* 80 */ 220, 218, 269, 314, 327, 283, 308, 336, 326, 340, - /* 90 */ 352, 362, 147, 415, 405, 393, 409, 386, 401, 406, - /* 100 */ 184, 477, 460, 464, 466, 490, 486, 491, 495, 502, - /* 110 */ 508, 499, 509, 496, 506, 510, 524, 516, 518, 519, - /* 120 */ 526, + /* 0 */ 505, -6, 49, 109, 127, 155, 188, 216, 290, 328, + /* 10 */ 343, 410, 429, 254, 509, 484, 547, 565, 580, 591, + /* 20 */ 623, 626, 651, 662, 690, 695, 706, 734, 752, 767, + /* 30 */ 778, 810, 813, 273, -146, 136, 241, 67, 296, 104, + /* 40 */ 110, -171, -170, -164, -73, -182, -102, -226, 15, 44, + /* 50 */ -54, 215, 249, 266, 96, -24, 268, 207, 375, 115, + /* 60 */ -150, -150, -150, -71, -162, -176, 97, -65, 131, 200, + /* 70 */ 270, 312, 313, 366, 368, 393, 397, 399, 401, 403, + /* 80 */ 412, 422, 435, 441, 452, 457, 485, 489, -48, 145, + /* 90 */ 156, 214, 303, 152, 310, 339, -27, 197, 344, 64, + /* 100 */ -175, -154, 262, 275, 294, 329, 349, 299, 272, 390, + /* 110 */ 385, 519, 483, 488, 488, 488, 535, 479, 499, 570, + /* 120 */ 568, 546, 562, 599, 574, 604, 600, 601, 637, 632, + /* 130 */ 644, 663, 646, 648, 649, 650, 659, 660, 661, 664, + /* 140 */ 667, 671, 673, 677, 683, 652, 681, 669, 656, 657, + /* 150 */ 629, 674, 668, 640, 678, 680, 653, 655, 670, 687, + /* 160 */ 488, 698, 691, 692, 675, 684, 686, 707, 535, 721, + /* 170 */ 719, 724, 728, 732, 735, 749, 739, 764, 776, 759, + /* 180 */ 768, 782, 785, 790, 781, 783, 789, 801, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 10 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 20 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 30 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 40 */ 702, 702, 722, 702, 702, 702, 702, 702, 702, 702, - /* 50 */ 702, 702, 720, 702, 872, 702, 806, 702, 702, 883, - /* 60 */ 883, 883, 722, 720, 702, 702, 702, 783, 702, 947, - /* 70 */ 702, 907, 899, 875, 889, 876, 702, 932, 892, 702, - /* 80 */ 702, 914, 912, 702, 702, 914, 912, 702, 926, 922, - /* 90 */ 905, 903, 889, 702, 702, 702, 702, 950, 938, 934, - /* 100 */ 702, 702, 811, 808, 808, 702, 722, 702, 722, 702, - /* 110 */ 702, 702, 720, 702, 752, 702, 720, 702, 786, 786, - /* 120 */ 723, 702, 702, 702, 702, 925, 924, 849, 848, 847, - /* 130 */ 843, 844, 702, 702, 702, 702, 702, 702, 838, 839, - /* 140 */ 837, 836, 702, 702, 702, 873, 702, 702, 702, 935, - /* 150 */ 939, 702, 825, 702, 702, 702, 702, 702, 702, 702, - /* 160 */ 702, 702, 896, 906, 702, 702, 702, 702, 702, 702, - /* 170 */ 702, 702, 702, 702, 702, 702, 702, 825, 702, 923, - /* 180 */ 702, 882, 878, 702, 702, 874, 702, 868, 702, 702, - /* 190 */ 702, 933, 702, 702, 702, 702, 702, 702, 702, 702, - /* 200 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 210 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 220 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 230 */ 702, 702, 824, 702, 702, 702, 702, 702, 702, 702, - /* 240 */ 780, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 250 */ 702, 702, 702, 765, 763, 762, 761, 702, 758, 702, - /* 260 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 270 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 280 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 0 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 10 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 20 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 30 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 40 */ 971, 971, 971, 1024, 971, 971, 971, 971, 971, 971, + /* 50 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 1022, + /* 60 */ 971, 1219, 971, 1136, 971, 971, 971, 971, 971, 971, + /* 70 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 80 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 1024, + /* 90 */ 1230, 1230, 1230, 1022, 971, 971, 971, 971, 971, 1107, + /* 100 */ 971, 971, 971, 971, 971, 971, 971, 1294, 971, 1060, + /* 110 */ 1254, 971, 1246, 1222, 1236, 1223, 971, 1279, 1239, 971, + /* 120 */ 1141, 1138, 1138, 971, 971, 1024, 971, 971, 1024, 971, + /* 130 */ 1024, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 140 */ 971, 971, 971, 971, 971, 971, 1022, 971, 971, 971, + /* 150 */ 1261, 1259, 971, 1261, 1259, 971, 1273, 1269, 1252, 1250, + /* 160 */ 1236, 971, 971, 971, 1297, 1285, 1281, 971, 971, 1259, + /* 170 */ 971, 971, 1259, 971, 1149, 971, 971, 971, 1022, 971, + /* 180 */ 1076, 971, 1022, 971, 1110, 1110, 1025, 976, 971, 971, + /* 190 */ 971, 971, 971, 971, 971, 971, 971, 971, 1191, 1272, + /* 200 */ 1271, 1190, 1196, 1195, 1194, 971, 971, 971, 1185, 1186, + /* 210 */ 1184, 1183, 971, 971, 971, 971, 971, 971, 971, 971, + /* 220 */ 971, 971, 971, 1220, 971, 1282, 1286, 971, 971, 971, + /* 230 */ 1170, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 240 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 250 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 260 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 270 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 280 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 290 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 300 */ 971, 971, 971, 971, 971, 971, 1243, 1253, 971, 971, + /* 310 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 1170, + /* 320 */ 971, 1270, 971, 1229, 1225, 971, 971, 1221, 971, 971, + /* 330 */ 1280, 971, 971, 971, 971, 971, 971, 971, 971, 1215, + /* 340 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 350 */ 971, 971, 971, 971, 971, 971, 1169, 971, 971, 971, + /* 360 */ 971, 971, 971, 1104, 971, 971, 971, 971, 971, 971, + /* 370 */ 971, 971, 971, 971, 971, 971, 1089, 1087, 1086, 1085, + /* 380 */ 971, 1082, 971, 971, 971, 971, 971, 971, 971, 971, + /* 390 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 400 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 410 */ 971, 971, 971, 971, 971, 971, 971, 971, 971, 971, + /* 420 */ 971, 971, 971, 971, }; /********** End of lemon-generated parsing tables *****************************/ @@ -619,205 +714,243 @@ static const char *const yyTokenName[] = { /* 16 */ "NK_REM", /* 17 */ "NK_CONCAT", /* 18 */ "CREATE", - /* 19 */ "USER", - /* 20 */ "PASS", - /* 21 */ "NK_STRING", - /* 22 */ "ALTER", - /* 23 */ "PRIVILEGE", - /* 24 */ "DROP", - /* 25 */ "DNODE", - /* 26 */ "PORT", - /* 27 */ "NK_INTEGER", - /* 28 */ "NK_ID", - /* 29 */ "NK_IPTOKEN", - /* 30 */ "DATABASE", - /* 31 */ "USE", - /* 32 */ "IF", - /* 33 */ "NOT", - /* 34 */ "EXISTS", - /* 35 */ "BLOCKS", - /* 36 */ "CACHE", - /* 37 */ "CACHELAST", - /* 38 */ "COMP", - /* 39 */ "DAYS", - /* 40 */ "FSYNC", - /* 41 */ "MAXROWS", - /* 42 */ "MINROWS", - /* 43 */ "KEEP", - /* 44 */ "PRECISION", - /* 45 */ "QUORUM", - /* 46 */ "REPLICA", - /* 47 */ "TTL", - /* 48 */ "WAL", - /* 49 */ "VGROUPS", - /* 50 */ "SINGLE_STABLE", - /* 51 */ "STREAM_MODE", - /* 52 */ "TABLE", - /* 53 */ "NK_LP", - /* 54 */ "NK_RP", - /* 55 */ "STABLE", - /* 56 */ "USING", - /* 57 */ "TAGS", - /* 58 */ "NK_DOT", - /* 59 */ "NK_COMMA", - /* 60 */ "COMMENT", - /* 61 */ "BOOL", - /* 62 */ "TINYINT", - /* 63 */ "SMALLINT", - /* 64 */ "INT", - /* 65 */ "INTEGER", - /* 66 */ "BIGINT", - /* 67 */ "FLOAT", - /* 68 */ "DOUBLE", - /* 69 */ "BINARY", - /* 70 */ "TIMESTAMP", - /* 71 */ "NCHAR", - /* 72 */ "UNSIGNED", - /* 73 */ "JSON", - /* 74 */ "VARCHAR", - /* 75 */ "MEDIUMBLOB", - /* 76 */ "BLOB", - /* 77 */ "VARBINARY", - /* 78 */ "DECIMAL", - /* 79 */ "SMA", - /* 80 */ "SHOW", - /* 81 */ "DNODES", - /* 82 */ "USERS", - /* 83 */ "DATABASES", - /* 84 */ "TABLES", - /* 85 */ "STABLES", - /* 86 */ "MNODES", - /* 87 */ "MODULES", - /* 88 */ "QNODES", - /* 89 */ "FUNCTIONS", - /* 90 */ "INDEXES", - /* 91 */ "FROM", - /* 92 */ "STREAMS", - /* 93 */ "LIKE", - /* 94 */ "NK_FLOAT", - /* 95 */ "NK_BOOL", - /* 96 */ "NK_VARIABLE", - /* 97 */ "BETWEEN", - /* 98 */ "IS", - /* 99 */ "NULL", - /* 100 */ "NK_LT", - /* 101 */ "NK_GT", - /* 102 */ "NK_LE", - /* 103 */ "NK_GE", - /* 104 */ "NK_NE", - /* 105 */ "NK_EQ", - /* 106 */ "MATCH", - /* 107 */ "NMATCH", - /* 108 */ "IN", - /* 109 */ "AS", - /* 110 */ "JOIN", - /* 111 */ "ON", - /* 112 */ "INNER", - /* 113 */ "SELECT", - /* 114 */ "DISTINCT", - /* 115 */ "WHERE", - /* 116 */ "PARTITION", - /* 117 */ "BY", - /* 118 */ "SESSION", - /* 119 */ "STATE_WINDOW", - /* 120 */ "INTERVAL", - /* 121 */ "SLIDING", - /* 122 */ "FILL", - /* 123 */ "VALUE", - /* 124 */ "NONE", - /* 125 */ "PREV", - /* 126 */ "LINEAR", - /* 127 */ "NEXT", - /* 128 */ "GROUP", - /* 129 */ "HAVING", - /* 130 */ "ORDER", - /* 131 */ "SLIMIT", - /* 132 */ "SOFFSET", - /* 133 */ "LIMIT", - /* 134 */ "OFFSET", - /* 135 */ "ASC", - /* 136 */ "DESC", - /* 137 */ "NULLS", - /* 138 */ "FIRST", - /* 139 */ "LAST", - /* 140 */ "cmd", - /* 141 */ "user_name", - /* 142 */ "dnode_endpoint", - /* 143 */ "dnode_host_name", - /* 144 */ "not_exists_opt", - /* 145 */ "db_name", - /* 146 */ "db_options", - /* 147 */ "exists_opt", - /* 148 */ "full_table_name", - /* 149 */ "column_def_list", - /* 150 */ "tags_def_opt", - /* 151 */ "table_options", - /* 152 */ "multi_create_clause", - /* 153 */ "tags_def", - /* 154 */ "multi_drop_clause", - /* 155 */ "create_subtable_clause", - /* 156 */ "specific_tags_opt", - /* 157 */ "literal_list", - /* 158 */ "drop_table_clause", - /* 159 */ "col_name_list", - /* 160 */ "table_name", - /* 161 */ "column_def", - /* 162 */ "column_name", - /* 163 */ "type_name", - /* 164 */ "col_name", - /* 165 */ "db_name_cond_opt", - /* 166 */ "like_pattern_opt", - /* 167 */ "table_name_cond", - /* 168 */ "from_db_opt", - /* 169 */ "query_expression", - /* 170 */ "literal", - /* 171 */ "duration_literal", - /* 172 */ "function_name", - /* 173 */ "table_alias", - /* 174 */ "column_alias", - /* 175 */ "expression", - /* 176 */ "column_reference", - /* 177 */ "expression_list", - /* 178 */ "subquery", - /* 179 */ "predicate", - /* 180 */ "compare_op", - /* 181 */ "in_op", - /* 182 */ "in_predicate_value", - /* 183 */ "boolean_value_expression", - /* 184 */ "boolean_primary", - /* 185 */ "common_expression", - /* 186 */ "from_clause", - /* 187 */ "table_reference_list", - /* 188 */ "table_reference", - /* 189 */ "table_primary", - /* 190 */ "joined_table", - /* 191 */ "alias_opt", - /* 192 */ "parenthesized_joined_table", - /* 193 */ "join_type", - /* 194 */ "search_condition", - /* 195 */ "query_specification", - /* 196 */ "set_quantifier_opt", - /* 197 */ "select_list", - /* 198 */ "where_clause_opt", - /* 199 */ "partition_by_clause_opt", - /* 200 */ "twindow_clause_opt", - /* 201 */ "group_by_clause_opt", - /* 202 */ "having_clause_opt", - /* 203 */ "select_sublist", - /* 204 */ "select_item", - /* 205 */ "sliding_opt", - /* 206 */ "fill_opt", - /* 207 */ "fill_mode", - /* 208 */ "group_by_list", - /* 209 */ "query_expression_body", - /* 210 */ "order_by_clause_opt", - /* 211 */ "slimit_clause_opt", - /* 212 */ "limit_clause_opt", - /* 213 */ "query_primary", - /* 214 */ "sort_specification_list", - /* 215 */ "sort_specification", - /* 216 */ "ordering_specification_opt", - /* 217 */ "null_ordering_opt", + /* 19 */ "ACCOUNT", + /* 20 */ "NK_ID", + /* 21 */ "PASS", + /* 22 */ "NK_STRING", + /* 23 */ "ALTER", + /* 24 */ "PPS", + /* 25 */ "TSERIES", + /* 26 */ "STORAGE", + /* 27 */ "STREAMS", + /* 28 */ "QTIME", + /* 29 */ "DBS", + /* 30 */ "USERS", + /* 31 */ "CONNS", + /* 32 */ "STATE", + /* 33 */ "USER", + /* 34 */ "PRIVILEGE", + /* 35 */ "DROP", + /* 36 */ "DNODE", + /* 37 */ "PORT", + /* 38 */ "NK_INTEGER", + /* 39 */ "DNODES", + /* 40 */ "NK_IPTOKEN", + /* 41 */ "LOCAL", + /* 42 */ "QNODE", + /* 43 */ "ON", + /* 44 */ "DATABASE", + /* 45 */ "USE", + /* 46 */ "IF", + /* 47 */ "NOT", + /* 48 */ "EXISTS", + /* 49 */ "BLOCKS", + /* 50 */ "CACHE", + /* 51 */ "CACHELAST", + /* 52 */ "COMP", + /* 53 */ "DAYS", + /* 54 */ "FSYNC", + /* 55 */ "MAXROWS", + /* 56 */ "MINROWS", + /* 57 */ "KEEP", + /* 58 */ "PRECISION", + /* 59 */ "QUORUM", + /* 60 */ "REPLICA", + /* 61 */ "TTL", + /* 62 */ "WAL", + /* 63 */ "VGROUPS", + /* 64 */ "SINGLE_STABLE", + /* 65 */ "STREAM_MODE", + /* 66 */ "RETENTIONS", + /* 67 */ "FILE_FACTOR", + /* 68 */ "NK_FLOAT", + /* 69 */ "TABLE", + /* 70 */ "NK_LP", + /* 71 */ "NK_RP", + /* 72 */ "STABLE", + /* 73 */ "ADD", + /* 74 */ "COLUMN", + /* 75 */ "MODIFY", + /* 76 */ "RENAME", + /* 77 */ "TAG", + /* 78 */ "SET", + /* 79 */ "NK_EQ", + /* 80 */ "USING", + /* 81 */ "TAGS", + /* 82 */ "NK_DOT", + /* 83 */ "NK_COMMA", + /* 84 */ "COMMENT", + /* 85 */ "BOOL", + /* 86 */ "TINYINT", + /* 87 */ "SMALLINT", + /* 88 */ "INT", + /* 89 */ "INTEGER", + /* 90 */ "BIGINT", + /* 91 */ "FLOAT", + /* 92 */ "DOUBLE", + /* 93 */ "BINARY", + /* 94 */ "TIMESTAMP", + /* 95 */ "NCHAR", + /* 96 */ "UNSIGNED", + /* 97 */ "JSON", + /* 98 */ "VARCHAR", + /* 99 */ "MEDIUMBLOB", + /* 100 */ "BLOB", + /* 101 */ "VARBINARY", + /* 102 */ "DECIMAL", + /* 103 */ "SMA", + /* 104 */ "ROLLUP", + /* 105 */ "SHOW", + /* 106 */ "DATABASES", + /* 107 */ "TABLES", + /* 108 */ "STABLES", + /* 109 */ "MNODES", + /* 110 */ "MODULES", + /* 111 */ "QNODES", + /* 112 */ "FUNCTIONS", + /* 113 */ "INDEXES", + /* 114 */ "FROM", + /* 115 */ "LIKE", + /* 116 */ "INDEX", + /* 117 */ "FULLTEXT", + /* 118 */ "FUNCTION", + /* 119 */ "INTERVAL", + /* 120 */ "TOPIC", + /* 121 */ "AS", + /* 122 */ "NK_BOOL", + /* 123 */ "NK_VARIABLE", + /* 124 */ "BETWEEN", + /* 125 */ "IS", + /* 126 */ "NULL", + /* 127 */ "NK_LT", + /* 128 */ "NK_GT", + /* 129 */ "NK_LE", + /* 130 */ "NK_GE", + /* 131 */ "NK_NE", + /* 132 */ "MATCH", + /* 133 */ "NMATCH", + /* 134 */ "IN", + /* 135 */ "JOIN", + /* 136 */ "INNER", + /* 137 */ "SELECT", + /* 138 */ "DISTINCT", + /* 139 */ "WHERE", + /* 140 */ "PARTITION", + /* 141 */ "BY", + /* 142 */ "SESSION", + /* 143 */ "STATE_WINDOW", + /* 144 */ "SLIDING", + /* 145 */ "FILL", + /* 146 */ "VALUE", + /* 147 */ "NONE", + /* 148 */ "PREV", + /* 149 */ "LINEAR", + /* 150 */ "NEXT", + /* 151 */ "GROUP", + /* 152 */ "HAVING", + /* 153 */ "ORDER", + /* 154 */ "SLIMIT", + /* 155 */ "SOFFSET", + /* 156 */ "LIMIT", + /* 157 */ "OFFSET", + /* 158 */ "ASC", + /* 159 */ "DESC", + /* 160 */ "NULLS", + /* 161 */ "FIRST", + /* 162 */ "LAST", + /* 163 */ "cmd", + /* 164 */ "account_options", + /* 165 */ "alter_account_options", + /* 166 */ "literal", + /* 167 */ "alter_account_option", + /* 168 */ "user_name", + /* 169 */ "dnode_endpoint", + /* 170 */ "dnode_host_name", + /* 171 */ "not_exists_opt", + /* 172 */ "db_name", + /* 173 */ "db_options", + /* 174 */ "exists_opt", + /* 175 */ "alter_db_options", + /* 176 */ "alter_db_option", + /* 177 */ "full_table_name", + /* 178 */ "column_def_list", + /* 179 */ "tags_def_opt", + /* 180 */ "table_options", + /* 181 */ "multi_create_clause", + /* 182 */ "tags_def", + /* 183 */ "multi_drop_clause", + /* 184 */ "alter_table_clause", + /* 185 */ "alter_table_options", + /* 186 */ "column_name", + /* 187 */ "type_name", + /* 188 */ "create_subtable_clause", + /* 189 */ "specific_tags_opt", + /* 190 */ "literal_list", + /* 191 */ "drop_table_clause", + /* 192 */ "col_name_list", + /* 193 */ "table_name", + /* 194 */ "column_def", + /* 195 */ "func_name_list", + /* 196 */ "alter_table_option", + /* 197 */ "col_name", + /* 198 */ "db_name_cond_opt", + /* 199 */ "like_pattern_opt", + /* 200 */ "table_name_cond", + /* 201 */ "from_db_opt", + /* 202 */ "func_name", + /* 203 */ "function_name", + /* 204 */ "index_name", + /* 205 */ "index_options", + /* 206 */ "func_list", + /* 207 */ "duration_literal", + /* 208 */ "sliding_opt", + /* 209 */ "func", + /* 210 */ "expression_list", + /* 211 */ "topic_name", + /* 212 */ "query_expression", + /* 213 */ "table_alias", + /* 214 */ "column_alias", + /* 215 */ "expression", + /* 216 */ "column_reference", + /* 217 */ "subquery", + /* 218 */ "predicate", + /* 219 */ "compare_op", + /* 220 */ "in_op", + /* 221 */ "in_predicate_value", + /* 222 */ "boolean_value_expression", + /* 223 */ "boolean_primary", + /* 224 */ "common_expression", + /* 225 */ "from_clause", + /* 226 */ "table_reference_list", + /* 227 */ "table_reference", + /* 228 */ "table_primary", + /* 229 */ "joined_table", + /* 230 */ "alias_opt", + /* 231 */ "parenthesized_joined_table", + /* 232 */ "join_type", + /* 233 */ "search_condition", + /* 234 */ "query_specification", + /* 235 */ "set_quantifier_opt", + /* 236 */ "select_list", + /* 237 */ "where_clause_opt", + /* 238 */ "partition_by_clause_opt", + /* 239 */ "twindow_clause_opt", + /* 240 */ "group_by_clause_opt", + /* 241 */ "having_clause_opt", + /* 242 */ "select_sublist", + /* 243 */ "select_item", + /* 244 */ "fill_opt", + /* 245 */ "fill_mode", + /* 246 */ "group_by_list", + /* 247 */ "query_expression_body", + /* 248 */ "order_by_clause_opt", + /* 249 */ "slimit_clause_opt", + /* 250 */ "limit_clause_opt", + /* 251 */ "query_primary", + /* 252 */ "sort_specification_list", + /* 253 */ "sort_specification", + /* 254 */ "ordering_specification_opt", + /* 255 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -825,254 +958,332 @@ static const char *const yyTokenName[] = { /* For tracing reduce actions, the names of all rules are required. */ static const char *const yyRuleName[] = { - /* 0 */ "cmd ::= CREATE USER user_name PASS NK_STRING", - /* 1 */ "cmd ::= ALTER USER user_name PASS NK_STRING", - /* 2 */ "cmd ::= ALTER USER user_name PRIVILEGE NK_STRING", - /* 3 */ "cmd ::= DROP USER user_name", - /* 4 */ "cmd ::= CREATE DNODE dnode_endpoint", - /* 5 */ "cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER", - /* 6 */ "cmd ::= DROP DNODE NK_INTEGER", - /* 7 */ "cmd ::= DROP DNODE dnode_endpoint", - /* 8 */ "dnode_endpoint ::= NK_STRING", - /* 9 */ "dnode_host_name ::= NK_ID", - /* 10 */ "dnode_host_name ::= NK_IPTOKEN", - /* 11 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", - /* 12 */ "cmd ::= DROP DATABASE exists_opt db_name", - /* 13 */ "cmd ::= USE db_name", - /* 14 */ "not_exists_opt ::= IF NOT EXISTS", - /* 15 */ "not_exists_opt ::=", - /* 16 */ "exists_opt ::= IF EXISTS", - /* 17 */ "exists_opt ::=", - /* 18 */ "db_options ::=", - /* 19 */ "db_options ::= db_options BLOCKS NK_INTEGER", - /* 20 */ "db_options ::= db_options CACHE NK_INTEGER", - /* 21 */ "db_options ::= db_options CACHELAST NK_INTEGER", - /* 22 */ "db_options ::= db_options COMP NK_INTEGER", - /* 23 */ "db_options ::= db_options DAYS NK_INTEGER", - /* 24 */ "db_options ::= db_options FSYNC NK_INTEGER", - /* 25 */ "db_options ::= db_options MAXROWS NK_INTEGER", - /* 26 */ "db_options ::= db_options MINROWS NK_INTEGER", - /* 27 */ "db_options ::= db_options KEEP NK_INTEGER", - /* 28 */ "db_options ::= db_options PRECISION NK_STRING", - /* 29 */ "db_options ::= db_options QUORUM NK_INTEGER", - /* 30 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 31 */ "db_options ::= db_options TTL NK_INTEGER", - /* 32 */ "db_options ::= db_options WAL NK_INTEGER", - /* 33 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 34 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 35 */ "db_options ::= db_options STREAM_MODE NK_INTEGER", - /* 36 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 37 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 38 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 39 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 40 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 41 */ "multi_create_clause ::= create_subtable_clause", - /* 42 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 43 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP", - /* 44 */ "multi_drop_clause ::= drop_table_clause", - /* 45 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", - /* 46 */ "drop_table_clause ::= exists_opt full_table_name", - /* 47 */ "specific_tags_opt ::=", - /* 48 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", - /* 49 */ "full_table_name ::= table_name", - /* 50 */ "full_table_name ::= db_name NK_DOT table_name", - /* 51 */ "column_def_list ::= column_def", - /* 52 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 53 */ "column_def ::= column_name type_name", - /* 54 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 55 */ "type_name ::= BOOL", - /* 56 */ "type_name ::= TINYINT", - /* 57 */ "type_name ::= SMALLINT", - /* 58 */ "type_name ::= INT", - /* 59 */ "type_name ::= INTEGER", - /* 60 */ "type_name ::= BIGINT", - /* 61 */ "type_name ::= FLOAT", - /* 62 */ "type_name ::= DOUBLE", - /* 63 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 64 */ "type_name ::= TIMESTAMP", - /* 65 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 66 */ "type_name ::= TINYINT UNSIGNED", - /* 67 */ "type_name ::= SMALLINT UNSIGNED", - /* 68 */ "type_name ::= INT UNSIGNED", - /* 69 */ "type_name ::= BIGINT UNSIGNED", - /* 70 */ "type_name ::= JSON", - /* 71 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 72 */ "type_name ::= MEDIUMBLOB", - /* 73 */ "type_name ::= BLOB", - /* 74 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 75 */ "type_name ::= DECIMAL", - /* 76 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 77 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 78 */ "tags_def_opt ::=", - /* 79 */ "tags_def_opt ::= tags_def", - /* 80 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 81 */ "table_options ::=", - /* 82 */ "table_options ::= table_options COMMENT NK_STRING", - /* 83 */ "table_options ::= table_options KEEP NK_INTEGER", - /* 84 */ "table_options ::= table_options TTL NK_INTEGER", - /* 85 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 86 */ "col_name_list ::= col_name", - /* 87 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 88 */ "col_name ::= column_name", - /* 89 */ "cmd ::= SHOW DNODES", - /* 90 */ "cmd ::= SHOW USERS", - /* 91 */ "cmd ::= SHOW DATABASES", - /* 92 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 93 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 94 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 95 */ "cmd ::= SHOW MNODES", - /* 96 */ "cmd ::= SHOW MODULES", - /* 97 */ "cmd ::= SHOW QNODES", - /* 98 */ "cmd ::= SHOW FUNCTIONS", - /* 99 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 100 */ "cmd ::= SHOW STREAMS", - /* 101 */ "db_name_cond_opt ::=", - /* 102 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 103 */ "like_pattern_opt ::=", - /* 104 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 105 */ "table_name_cond ::= table_name", - /* 106 */ "from_db_opt ::=", - /* 107 */ "from_db_opt ::= FROM db_name", - /* 108 */ "cmd ::= query_expression", - /* 109 */ "literal ::= NK_INTEGER", - /* 110 */ "literal ::= NK_FLOAT", - /* 111 */ "literal ::= NK_STRING", - /* 112 */ "literal ::= NK_BOOL", - /* 113 */ "literal ::= TIMESTAMP NK_STRING", - /* 114 */ "literal ::= duration_literal", - /* 115 */ "duration_literal ::= NK_VARIABLE", - /* 116 */ "literal_list ::= literal", - /* 117 */ "literal_list ::= literal_list NK_COMMA literal", - /* 118 */ "db_name ::= NK_ID", - /* 119 */ "table_name ::= NK_ID", - /* 120 */ "column_name ::= NK_ID", - /* 121 */ "function_name ::= NK_ID", - /* 122 */ "table_alias ::= NK_ID", - /* 123 */ "column_alias ::= NK_ID", - /* 124 */ "user_name ::= NK_ID", - /* 125 */ "expression ::= literal", - /* 126 */ "expression ::= column_reference", - /* 127 */ "expression ::= function_name NK_LP expression_list NK_RP", - /* 128 */ "expression ::= function_name NK_LP NK_STAR NK_RP", - /* 129 */ "expression ::= subquery", - /* 130 */ "expression ::= NK_LP expression NK_RP", - /* 131 */ "expression ::= NK_PLUS expression", - /* 132 */ "expression ::= NK_MINUS expression", - /* 133 */ "expression ::= expression NK_PLUS expression", - /* 134 */ "expression ::= expression NK_MINUS expression", - /* 135 */ "expression ::= expression NK_STAR expression", - /* 136 */ "expression ::= expression NK_SLASH expression", - /* 137 */ "expression ::= expression NK_REM expression", - /* 138 */ "expression_list ::= expression", - /* 139 */ "expression_list ::= expression_list NK_COMMA expression", - /* 140 */ "column_reference ::= column_name", - /* 141 */ "column_reference ::= table_name NK_DOT column_name", - /* 142 */ "predicate ::= expression compare_op expression", - /* 143 */ "predicate ::= expression BETWEEN expression AND expression", - /* 144 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 145 */ "predicate ::= expression IS NULL", - /* 146 */ "predicate ::= expression IS NOT NULL", - /* 147 */ "predicate ::= expression in_op in_predicate_value", - /* 148 */ "compare_op ::= NK_LT", - /* 149 */ "compare_op ::= NK_GT", - /* 150 */ "compare_op ::= NK_LE", - /* 151 */ "compare_op ::= NK_GE", - /* 152 */ "compare_op ::= NK_NE", - /* 153 */ "compare_op ::= NK_EQ", - /* 154 */ "compare_op ::= LIKE", - /* 155 */ "compare_op ::= NOT LIKE", - /* 156 */ "compare_op ::= MATCH", - /* 157 */ "compare_op ::= NMATCH", - /* 158 */ "in_op ::= IN", - /* 159 */ "in_op ::= NOT IN", - /* 160 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 161 */ "boolean_value_expression ::= boolean_primary", - /* 162 */ "boolean_value_expression ::= NOT boolean_primary", - /* 163 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 164 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 165 */ "boolean_primary ::= predicate", - /* 166 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 167 */ "common_expression ::= expression", - /* 168 */ "common_expression ::= boolean_value_expression", - /* 169 */ "from_clause ::= FROM table_reference_list", - /* 170 */ "table_reference_list ::= table_reference", - /* 171 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 172 */ "table_reference ::= table_primary", - /* 173 */ "table_reference ::= joined_table", - /* 174 */ "table_primary ::= table_name alias_opt", - /* 175 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 176 */ "table_primary ::= subquery alias_opt", - /* 177 */ "table_primary ::= parenthesized_joined_table", - /* 178 */ "alias_opt ::=", - /* 179 */ "alias_opt ::= table_alias", - /* 180 */ "alias_opt ::= AS table_alias", - /* 181 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 182 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 183 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 184 */ "join_type ::=", - /* 185 */ "join_type ::= INNER", - /* 186 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 187 */ "set_quantifier_opt ::=", - /* 188 */ "set_quantifier_opt ::= DISTINCT", - /* 189 */ "set_quantifier_opt ::= ALL", - /* 190 */ "select_list ::= NK_STAR", - /* 191 */ "select_list ::= select_sublist", - /* 192 */ "select_sublist ::= select_item", - /* 193 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 194 */ "select_item ::= common_expression", - /* 195 */ "select_item ::= common_expression column_alias", - /* 196 */ "select_item ::= common_expression AS column_alias", - /* 197 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 198 */ "where_clause_opt ::=", - /* 199 */ "where_clause_opt ::= WHERE search_condition", - /* 200 */ "partition_by_clause_opt ::=", - /* 201 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 202 */ "twindow_clause_opt ::=", - /* 203 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", - /* 204 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", - /* 205 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 206 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 207 */ "sliding_opt ::=", - /* 208 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 209 */ "fill_opt ::=", - /* 210 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 211 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 212 */ "fill_mode ::= NONE", - /* 213 */ "fill_mode ::= PREV", - /* 214 */ "fill_mode ::= NULL", - /* 215 */ "fill_mode ::= LINEAR", - /* 216 */ "fill_mode ::= NEXT", - /* 217 */ "group_by_clause_opt ::=", - /* 218 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 219 */ "group_by_list ::= expression", - /* 220 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 221 */ "having_clause_opt ::=", - /* 222 */ "having_clause_opt ::= HAVING search_condition", - /* 223 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 224 */ "query_expression_body ::= query_primary", - /* 225 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 226 */ "query_primary ::= query_specification", - /* 227 */ "order_by_clause_opt ::=", - /* 228 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 229 */ "slimit_clause_opt ::=", - /* 230 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 231 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 232 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 233 */ "limit_clause_opt ::=", - /* 234 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 235 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 236 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 237 */ "subquery ::= NK_LP query_expression NK_RP", - /* 238 */ "search_condition ::= common_expression", - /* 239 */ "sort_specification_list ::= sort_specification", - /* 240 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 241 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 242 */ "ordering_specification_opt ::=", - /* 243 */ "ordering_specification_opt ::= ASC", - /* 244 */ "ordering_specification_opt ::= DESC", - /* 245 */ "null_ordering_opt ::=", - /* 246 */ "null_ordering_opt ::= NULLS FIRST", - /* 247 */ "null_ordering_opt ::= NULLS LAST", + /* 0 */ "cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options", + /* 1 */ "cmd ::= ALTER ACCOUNT NK_ID alter_account_options", + /* 2 */ "account_options ::=", + /* 3 */ "account_options ::= account_options PPS literal", + /* 4 */ "account_options ::= account_options TSERIES literal", + /* 5 */ "account_options ::= account_options STORAGE literal", + /* 6 */ "account_options ::= account_options STREAMS literal", + /* 7 */ "account_options ::= account_options QTIME literal", + /* 8 */ "account_options ::= account_options DBS literal", + /* 9 */ "account_options ::= account_options USERS literal", + /* 10 */ "account_options ::= account_options CONNS literal", + /* 11 */ "account_options ::= account_options STATE literal", + /* 12 */ "alter_account_options ::= alter_account_option", + /* 13 */ "alter_account_options ::= alter_account_options alter_account_option", + /* 14 */ "alter_account_option ::= PASS literal", + /* 15 */ "alter_account_option ::= PPS literal", + /* 16 */ "alter_account_option ::= TSERIES literal", + /* 17 */ "alter_account_option ::= STORAGE literal", + /* 18 */ "alter_account_option ::= STREAMS literal", + /* 19 */ "alter_account_option ::= QTIME literal", + /* 20 */ "alter_account_option ::= DBS literal", + /* 21 */ "alter_account_option ::= USERS literal", + /* 22 */ "alter_account_option ::= CONNS literal", + /* 23 */ "alter_account_option ::= STATE literal", + /* 24 */ "cmd ::= CREATE USER user_name PASS NK_STRING", + /* 25 */ "cmd ::= ALTER USER user_name PASS NK_STRING", + /* 26 */ "cmd ::= ALTER USER user_name PRIVILEGE NK_STRING", + /* 27 */ "cmd ::= DROP USER user_name", + /* 28 */ "cmd ::= CREATE DNODE dnode_endpoint", + /* 29 */ "cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER", + /* 30 */ "cmd ::= DROP DNODE NK_INTEGER", + /* 31 */ "cmd ::= DROP DNODE dnode_endpoint", + /* 32 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", + /* 33 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", + /* 34 */ "cmd ::= ALTER ALL DNODES NK_STRING", + /* 35 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", + /* 36 */ "dnode_endpoint ::= NK_STRING", + /* 37 */ "dnode_host_name ::= NK_ID", + /* 38 */ "dnode_host_name ::= NK_IPTOKEN", + /* 39 */ "cmd ::= ALTER LOCAL NK_STRING", + /* 40 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", + /* 41 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", + /* 42 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", + /* 43 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", + /* 44 */ "cmd ::= DROP DATABASE exists_opt db_name", + /* 45 */ "cmd ::= USE db_name", + /* 46 */ "cmd ::= ALTER DATABASE db_name alter_db_options", + /* 47 */ "not_exists_opt ::= IF NOT EXISTS", + /* 48 */ "not_exists_opt ::=", + /* 49 */ "exists_opt ::= IF EXISTS", + /* 50 */ "exists_opt ::=", + /* 51 */ "db_options ::=", + /* 52 */ "db_options ::= db_options BLOCKS NK_INTEGER", + /* 53 */ "db_options ::= db_options CACHE NK_INTEGER", + /* 54 */ "db_options ::= db_options CACHELAST NK_INTEGER", + /* 55 */ "db_options ::= db_options COMP NK_INTEGER", + /* 56 */ "db_options ::= db_options DAYS NK_INTEGER", + /* 57 */ "db_options ::= db_options FSYNC NK_INTEGER", + /* 58 */ "db_options ::= db_options MAXROWS NK_INTEGER", + /* 59 */ "db_options ::= db_options MINROWS NK_INTEGER", + /* 60 */ "db_options ::= db_options KEEP NK_INTEGER", + /* 61 */ "db_options ::= db_options PRECISION NK_STRING", + /* 62 */ "db_options ::= db_options QUORUM NK_INTEGER", + /* 63 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 64 */ "db_options ::= db_options TTL NK_INTEGER", + /* 65 */ "db_options ::= db_options WAL NK_INTEGER", + /* 66 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 67 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 68 */ "db_options ::= db_options STREAM_MODE NK_INTEGER", + /* 69 */ "db_options ::= db_options RETENTIONS NK_STRING", + /* 70 */ "db_options ::= db_options FILE_FACTOR NK_FLOAT", + /* 71 */ "alter_db_options ::= alter_db_option", + /* 72 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 73 */ "alter_db_option ::= BLOCKS NK_INTEGER", + /* 74 */ "alter_db_option ::= FSYNC NK_INTEGER", + /* 75 */ "alter_db_option ::= KEEP NK_INTEGER", + /* 76 */ "alter_db_option ::= WAL NK_INTEGER", + /* 77 */ "alter_db_option ::= QUORUM NK_INTEGER", + /* 78 */ "alter_db_option ::= CACHELAST NK_INTEGER", + /* 79 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 80 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 81 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 82 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 83 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 84 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 85 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 86 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 87 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 88 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 89 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 90 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 91 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 92 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 93 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 94 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 95 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal", + /* 96 */ "multi_create_clause ::= create_subtable_clause", + /* 97 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 98 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP", + /* 99 */ "multi_drop_clause ::= drop_table_clause", + /* 100 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", + /* 101 */ "drop_table_clause ::= exists_opt full_table_name", + /* 102 */ "specific_tags_opt ::=", + /* 103 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", + /* 104 */ "full_table_name ::= table_name", + /* 105 */ "full_table_name ::= db_name NK_DOT table_name", + /* 106 */ "column_def_list ::= column_def", + /* 107 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 108 */ "column_def ::= column_name type_name", + /* 109 */ "column_def ::= column_name type_name COMMENT NK_STRING", + /* 110 */ "type_name ::= BOOL", + /* 111 */ "type_name ::= TINYINT", + /* 112 */ "type_name ::= SMALLINT", + /* 113 */ "type_name ::= INT", + /* 114 */ "type_name ::= INTEGER", + /* 115 */ "type_name ::= BIGINT", + /* 116 */ "type_name ::= FLOAT", + /* 117 */ "type_name ::= DOUBLE", + /* 118 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 119 */ "type_name ::= TIMESTAMP", + /* 120 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 121 */ "type_name ::= TINYINT UNSIGNED", + /* 122 */ "type_name ::= SMALLINT UNSIGNED", + /* 123 */ "type_name ::= INT UNSIGNED", + /* 124 */ "type_name ::= BIGINT UNSIGNED", + /* 125 */ "type_name ::= JSON", + /* 126 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 127 */ "type_name ::= MEDIUMBLOB", + /* 128 */ "type_name ::= BLOB", + /* 129 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 130 */ "type_name ::= DECIMAL", + /* 131 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 132 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 133 */ "tags_def_opt ::=", + /* 134 */ "tags_def_opt ::= tags_def", + /* 135 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 136 */ "table_options ::=", + /* 137 */ "table_options ::= table_options COMMENT NK_STRING", + /* 138 */ "table_options ::= table_options KEEP NK_INTEGER", + /* 139 */ "table_options ::= table_options TTL NK_INTEGER", + /* 140 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 141 */ "table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP", + /* 142 */ "alter_table_options ::= alter_table_option", + /* 143 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 144 */ "alter_table_option ::= COMMENT NK_STRING", + /* 145 */ "alter_table_option ::= KEEP NK_INTEGER", + /* 146 */ "alter_table_option ::= TTL NK_INTEGER", + /* 147 */ "col_name_list ::= col_name", + /* 148 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 149 */ "col_name ::= column_name", + /* 150 */ "cmd ::= SHOW DNODES", + /* 151 */ "cmd ::= SHOW USERS", + /* 152 */ "cmd ::= SHOW DATABASES", + /* 153 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 154 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 155 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 156 */ "cmd ::= SHOW MNODES", + /* 157 */ "cmd ::= SHOW MODULES", + /* 158 */ "cmd ::= SHOW QNODES", + /* 159 */ "cmd ::= SHOW FUNCTIONS", + /* 160 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 161 */ "cmd ::= SHOW STREAMS", + /* 162 */ "db_name_cond_opt ::=", + /* 163 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 164 */ "like_pattern_opt ::=", + /* 165 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 166 */ "table_name_cond ::= table_name", + /* 167 */ "from_db_opt ::=", + /* 168 */ "from_db_opt ::= FROM db_name", + /* 169 */ "func_name_list ::= func_name", + /* 170 */ "func_name_list ::= func_name_list NK_COMMA col_name", + /* 171 */ "func_name ::= function_name", + /* 172 */ "cmd ::= CREATE SMA INDEX index_name ON table_name index_options", + /* 173 */ "cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP", + /* 174 */ "cmd ::= DROP INDEX index_name ON table_name", + /* 175 */ "index_options ::=", + /* 176 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", + /* 177 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", + /* 178 */ "func_list ::= func", + /* 179 */ "func_list ::= func_list NK_COMMA func", + /* 180 */ "func ::= function_name NK_LP expression_list NK_RP", + /* 181 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", + /* 182 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name", + /* 183 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 184 */ "cmd ::= query_expression", + /* 185 */ "literal ::= NK_INTEGER", + /* 186 */ "literal ::= NK_FLOAT", + /* 187 */ "literal ::= NK_STRING", + /* 188 */ "literal ::= NK_BOOL", + /* 189 */ "literal ::= TIMESTAMP NK_STRING", + /* 190 */ "literal ::= duration_literal", + /* 191 */ "duration_literal ::= NK_VARIABLE", + /* 192 */ "literal_list ::= literal", + /* 193 */ "literal_list ::= literal_list NK_COMMA literal", + /* 194 */ "db_name ::= NK_ID", + /* 195 */ "table_name ::= NK_ID", + /* 196 */ "column_name ::= NK_ID", + /* 197 */ "function_name ::= NK_ID", + /* 198 */ "table_alias ::= NK_ID", + /* 199 */ "column_alias ::= NK_ID", + /* 200 */ "user_name ::= NK_ID", + /* 201 */ "index_name ::= NK_ID", + /* 202 */ "topic_name ::= NK_ID", + /* 203 */ "expression ::= literal", + /* 204 */ "expression ::= column_reference", + /* 205 */ "expression ::= function_name NK_LP expression_list NK_RP", + /* 206 */ "expression ::= function_name NK_LP NK_STAR NK_RP", + /* 207 */ "expression ::= subquery", + /* 208 */ "expression ::= NK_LP expression NK_RP", + /* 209 */ "expression ::= NK_PLUS expression", + /* 210 */ "expression ::= NK_MINUS expression", + /* 211 */ "expression ::= expression NK_PLUS expression", + /* 212 */ "expression ::= expression NK_MINUS expression", + /* 213 */ "expression ::= expression NK_STAR expression", + /* 214 */ "expression ::= expression NK_SLASH expression", + /* 215 */ "expression ::= expression NK_REM expression", + /* 216 */ "expression_list ::= expression", + /* 217 */ "expression_list ::= expression_list NK_COMMA expression", + /* 218 */ "column_reference ::= column_name", + /* 219 */ "column_reference ::= table_name NK_DOT column_name", + /* 220 */ "predicate ::= expression compare_op expression", + /* 221 */ "predicate ::= expression BETWEEN expression AND expression", + /* 222 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 223 */ "predicate ::= expression IS NULL", + /* 224 */ "predicate ::= expression IS NOT NULL", + /* 225 */ "predicate ::= expression in_op in_predicate_value", + /* 226 */ "compare_op ::= NK_LT", + /* 227 */ "compare_op ::= NK_GT", + /* 228 */ "compare_op ::= NK_LE", + /* 229 */ "compare_op ::= NK_GE", + /* 230 */ "compare_op ::= NK_NE", + /* 231 */ "compare_op ::= NK_EQ", + /* 232 */ "compare_op ::= LIKE", + /* 233 */ "compare_op ::= NOT LIKE", + /* 234 */ "compare_op ::= MATCH", + /* 235 */ "compare_op ::= NMATCH", + /* 236 */ "in_op ::= IN", + /* 237 */ "in_op ::= NOT IN", + /* 238 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 239 */ "boolean_value_expression ::= boolean_primary", + /* 240 */ "boolean_value_expression ::= NOT boolean_primary", + /* 241 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 242 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 243 */ "boolean_primary ::= predicate", + /* 244 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 245 */ "common_expression ::= expression", + /* 246 */ "common_expression ::= boolean_value_expression", + /* 247 */ "from_clause ::= FROM table_reference_list", + /* 248 */ "table_reference_list ::= table_reference", + /* 249 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 250 */ "table_reference ::= table_primary", + /* 251 */ "table_reference ::= joined_table", + /* 252 */ "table_primary ::= table_name alias_opt", + /* 253 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 254 */ "table_primary ::= subquery alias_opt", + /* 255 */ "table_primary ::= parenthesized_joined_table", + /* 256 */ "alias_opt ::=", + /* 257 */ "alias_opt ::= table_alias", + /* 258 */ "alias_opt ::= AS table_alias", + /* 259 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 260 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 261 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 262 */ "join_type ::=", + /* 263 */ "join_type ::= INNER", + /* 264 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 265 */ "set_quantifier_opt ::=", + /* 266 */ "set_quantifier_opt ::= DISTINCT", + /* 267 */ "set_quantifier_opt ::= ALL", + /* 268 */ "select_list ::= NK_STAR", + /* 269 */ "select_list ::= select_sublist", + /* 270 */ "select_sublist ::= select_item", + /* 271 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 272 */ "select_item ::= common_expression", + /* 273 */ "select_item ::= common_expression column_alias", + /* 274 */ "select_item ::= common_expression AS column_alias", + /* 275 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 276 */ "where_clause_opt ::=", + /* 277 */ "where_clause_opt ::= WHERE search_condition", + /* 278 */ "partition_by_clause_opt ::=", + /* 279 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 280 */ "twindow_clause_opt ::=", + /* 281 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 282 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 283 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 284 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 285 */ "sliding_opt ::=", + /* 286 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 287 */ "fill_opt ::=", + /* 288 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 289 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 290 */ "fill_mode ::= NONE", + /* 291 */ "fill_mode ::= PREV", + /* 292 */ "fill_mode ::= NULL", + /* 293 */ "fill_mode ::= LINEAR", + /* 294 */ "fill_mode ::= NEXT", + /* 295 */ "group_by_clause_opt ::=", + /* 296 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 297 */ "group_by_list ::= expression", + /* 298 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 299 */ "having_clause_opt ::=", + /* 300 */ "having_clause_opt ::= HAVING search_condition", + /* 301 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 302 */ "query_expression_body ::= query_primary", + /* 303 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 304 */ "query_primary ::= query_specification", + /* 305 */ "order_by_clause_opt ::=", + /* 306 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 307 */ "slimit_clause_opt ::=", + /* 308 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 309 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 310 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 311 */ "limit_clause_opt ::=", + /* 312 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 313 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 314 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 315 */ "subquery ::= NK_LP query_expression NK_RP", + /* 316 */ "search_condition ::= common_expression", + /* 317 */ "sort_specification_list ::= sort_specification", + /* 318 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 319 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 320 */ "ordering_specification_opt ::=", + /* 321 */ "ordering_specification_opt ::= ASC", + /* 322 */ "ordering_specification_opt ::= DESC", + /* 323 */ "null_ordering_opt ::=", + /* 324 */ "null_ordering_opt ::= NULLS FIRST", + /* 325 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -1199,128 +1410,143 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 140: /* cmd */ - case 148: /* full_table_name */ - case 155: /* create_subtable_clause */ - case 158: /* drop_table_clause */ - case 161: /* column_def */ - case 164: /* col_name */ - case 165: /* db_name_cond_opt */ - case 166: /* like_pattern_opt */ - case 167: /* table_name_cond */ - case 168: /* from_db_opt */ - case 169: /* query_expression */ - case 170: /* literal */ - case 171: /* duration_literal */ - case 175: /* expression */ - case 176: /* column_reference */ - case 178: /* subquery */ - case 179: /* predicate */ - case 182: /* in_predicate_value */ - case 183: /* boolean_value_expression */ - case 184: /* boolean_primary */ - case 185: /* common_expression */ - case 186: /* from_clause */ - case 187: /* table_reference_list */ - case 188: /* table_reference */ - case 189: /* table_primary */ - case 190: /* joined_table */ - case 192: /* parenthesized_joined_table */ - case 194: /* search_condition */ - case 195: /* query_specification */ - case 198: /* where_clause_opt */ - case 200: /* twindow_clause_opt */ - case 202: /* having_clause_opt */ - case 204: /* select_item */ - case 205: /* sliding_opt */ - case 206: /* fill_opt */ - case 209: /* query_expression_body */ - case 211: /* slimit_clause_opt */ - case 212: /* limit_clause_opt */ - case 213: /* query_primary */ - case 215: /* sort_specification */ + case 163: /* cmd */ + case 166: /* literal */ + case 173: /* db_options */ + case 175: /* alter_db_options */ + case 177: /* full_table_name */ + case 180: /* table_options */ + case 184: /* alter_table_clause */ + case 185: /* alter_table_options */ + case 188: /* create_subtable_clause */ + case 191: /* drop_table_clause */ + case 194: /* column_def */ + case 197: /* col_name */ + case 198: /* db_name_cond_opt */ + case 199: /* like_pattern_opt */ + case 200: /* table_name_cond */ + case 201: /* from_db_opt */ + case 202: /* func_name */ + case 205: /* index_options */ + case 207: /* duration_literal */ + case 208: /* sliding_opt */ + case 209: /* func */ + case 212: /* query_expression */ + case 215: /* expression */ + case 216: /* column_reference */ + case 217: /* subquery */ + case 218: /* predicate */ + case 221: /* in_predicate_value */ + case 222: /* boolean_value_expression */ + case 223: /* boolean_primary */ + case 224: /* common_expression */ + case 225: /* from_clause */ + case 226: /* table_reference_list */ + case 227: /* table_reference */ + case 228: /* table_primary */ + case 229: /* joined_table */ + case 231: /* parenthesized_joined_table */ + case 233: /* search_condition */ + case 234: /* query_specification */ + case 237: /* where_clause_opt */ + case 239: /* twindow_clause_opt */ + case 241: /* having_clause_opt */ + case 243: /* select_item */ + case 244: /* fill_opt */ + case 247: /* query_expression_body */ + case 249: /* slimit_clause_opt */ + case 250: /* limit_clause_opt */ + case 251: /* query_primary */ + case 253: /* sort_specification */ { - nodesDestroyNode((yypminor->yy68)); + nodesDestroyNode((yypminor->yy360)); } break; - case 141: /* user_name */ - case 142: /* dnode_endpoint */ - case 143: /* dnode_host_name */ - case 145: /* db_name */ - case 160: /* table_name */ - case 162: /* column_name */ - case 172: /* function_name */ - case 173: /* table_alias */ - case 174: /* column_alias */ - case 191: /* alias_opt */ + case 164: /* account_options */ + case 165: /* alter_account_options */ + case 167: /* alter_account_option */ { } break; - case 144: /* not_exists_opt */ - case 147: /* exists_opt */ - case 196: /* set_quantifier_opt */ + case 168: /* user_name */ + case 169: /* dnode_endpoint */ + case 170: /* dnode_host_name */ + case 172: /* db_name */ + case 186: /* column_name */ + case 193: /* table_name */ + case 203: /* function_name */ + case 204: /* index_name */ + case 211: /* topic_name */ + case 213: /* table_alias */ + case 214: /* column_alias */ + case 230: /* alias_opt */ { } break; - case 146: /* db_options */ + case 171: /* not_exists_opt */ + case 174: /* exists_opt */ + case 235: /* set_quantifier_opt */ { - tfree((yypminor->yy339)); + } break; - case 149: /* column_def_list */ - case 150: /* tags_def_opt */ - case 152: /* multi_create_clause */ - case 153: /* tags_def */ - case 154: /* multi_drop_clause */ - case 156: /* specific_tags_opt */ - case 157: /* literal_list */ - case 159: /* col_name_list */ - case 177: /* expression_list */ - case 197: /* select_list */ - case 199: /* partition_by_clause_opt */ - case 201: /* group_by_clause_opt */ - case 203: /* select_sublist */ - case 208: /* group_by_list */ - case 210: /* order_by_clause_opt */ - case 214: /* sort_specification_list */ + case 176: /* alter_db_option */ + case 196: /* alter_table_option */ { - nodesDestroyList((yypminor->yy40)); + } break; - case 151: /* table_options */ + case 178: /* column_def_list */ + case 179: /* tags_def_opt */ + case 181: /* multi_create_clause */ + case 182: /* tags_def */ + case 183: /* multi_drop_clause */ + case 189: /* specific_tags_opt */ + case 190: /* literal_list */ + case 192: /* col_name_list */ + case 195: /* func_name_list */ + case 206: /* func_list */ + case 210: /* expression_list */ + case 236: /* select_list */ + case 238: /* partition_by_clause_opt */ + case 240: /* group_by_clause_opt */ + case 242: /* select_sublist */ + case 246: /* group_by_list */ + case 248: /* order_by_clause_opt */ + case 252: /* sort_specification_list */ { - tfree((yypminor->yy418)); + nodesDestroyList((yypminor->yy24)); } break; - case 163: /* type_name */ + case 187: /* type_name */ { } break; - case 180: /* compare_op */ - case 181: /* in_op */ + case 219: /* compare_op */ + case 220: /* in_op */ { } break; - case 193: /* join_type */ + case 232: /* join_type */ { } break; - case 207: /* fill_mode */ + case 245: /* fill_mode */ { } break; - case 216: /* ordering_specification_opt */ + case 254: /* ordering_specification_opt */ { } break; - case 217: /* null_ordering_opt */ + case 255: /* null_ordering_opt */ { } @@ -1619,254 +1845,332 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 140, -5 }, /* (0) cmd ::= CREATE USER user_name PASS NK_STRING */ - { 140, -5 }, /* (1) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 140, -5 }, /* (2) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ - { 140, -3 }, /* (3) cmd ::= DROP USER user_name */ - { 140, -3 }, /* (4) cmd ::= CREATE DNODE dnode_endpoint */ - { 140, -5 }, /* (5) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ - { 140, -3 }, /* (6) cmd ::= DROP DNODE NK_INTEGER */ - { 140, -3 }, /* (7) cmd ::= DROP DNODE dnode_endpoint */ - { 142, -1 }, /* (8) dnode_endpoint ::= NK_STRING */ - { 143, -1 }, /* (9) dnode_host_name ::= NK_ID */ - { 143, -1 }, /* (10) dnode_host_name ::= NK_IPTOKEN */ - { 140, -5 }, /* (11) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 140, -4 }, /* (12) cmd ::= DROP DATABASE exists_opt db_name */ - { 140, -2 }, /* (13) cmd ::= USE db_name */ - { 144, -3 }, /* (14) not_exists_opt ::= IF NOT EXISTS */ - { 144, 0 }, /* (15) not_exists_opt ::= */ - { 147, -2 }, /* (16) exists_opt ::= IF EXISTS */ - { 147, 0 }, /* (17) exists_opt ::= */ - { 146, 0 }, /* (18) db_options ::= */ - { 146, -3 }, /* (19) db_options ::= db_options BLOCKS NK_INTEGER */ - { 146, -3 }, /* (20) db_options ::= db_options CACHE NK_INTEGER */ - { 146, -3 }, /* (21) db_options ::= db_options CACHELAST NK_INTEGER */ - { 146, -3 }, /* (22) db_options ::= db_options COMP NK_INTEGER */ - { 146, -3 }, /* (23) db_options ::= db_options DAYS NK_INTEGER */ - { 146, -3 }, /* (24) db_options ::= db_options FSYNC NK_INTEGER */ - { 146, -3 }, /* (25) db_options ::= db_options MAXROWS NK_INTEGER */ - { 146, -3 }, /* (26) db_options ::= db_options MINROWS NK_INTEGER */ - { 146, -3 }, /* (27) db_options ::= db_options KEEP NK_INTEGER */ - { 146, -3 }, /* (28) db_options ::= db_options PRECISION NK_STRING */ - { 146, -3 }, /* (29) db_options ::= db_options QUORUM NK_INTEGER */ - { 146, -3 }, /* (30) db_options ::= db_options REPLICA NK_INTEGER */ - { 146, -3 }, /* (31) db_options ::= db_options TTL NK_INTEGER */ - { 146, -3 }, /* (32) db_options ::= db_options WAL NK_INTEGER */ - { 146, -3 }, /* (33) db_options ::= db_options VGROUPS NK_INTEGER */ - { 146, -3 }, /* (34) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 146, -3 }, /* (35) db_options ::= db_options STREAM_MODE NK_INTEGER */ - { 140, -9 }, /* (36) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 140, -3 }, /* (37) cmd ::= CREATE TABLE multi_create_clause */ - { 140, -9 }, /* (38) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 140, -3 }, /* (39) cmd ::= DROP TABLE multi_drop_clause */ - { 140, -4 }, /* (40) cmd ::= DROP STABLE exists_opt full_table_name */ - { 152, -1 }, /* (41) multi_create_clause ::= create_subtable_clause */ - { 152, -2 }, /* (42) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 155, -9 }, /* (43) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ - { 154, -1 }, /* (44) multi_drop_clause ::= drop_table_clause */ - { 154, -2 }, /* (45) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 158, -2 }, /* (46) drop_table_clause ::= exists_opt full_table_name */ - { 156, 0 }, /* (47) specific_tags_opt ::= */ - { 156, -3 }, /* (48) specific_tags_opt ::= NK_LP col_name_list NK_RP */ - { 148, -1 }, /* (49) full_table_name ::= table_name */ - { 148, -3 }, /* (50) full_table_name ::= db_name NK_DOT table_name */ - { 149, -1 }, /* (51) column_def_list ::= column_def */ - { 149, -3 }, /* (52) column_def_list ::= column_def_list NK_COMMA column_def */ - { 161, -2 }, /* (53) column_def ::= column_name type_name */ - { 161, -4 }, /* (54) column_def ::= column_name type_name COMMENT NK_STRING */ - { 163, -1 }, /* (55) type_name ::= BOOL */ - { 163, -1 }, /* (56) type_name ::= TINYINT */ - { 163, -1 }, /* (57) type_name ::= SMALLINT */ - { 163, -1 }, /* (58) type_name ::= INT */ - { 163, -1 }, /* (59) type_name ::= INTEGER */ - { 163, -1 }, /* (60) type_name ::= BIGINT */ - { 163, -1 }, /* (61) type_name ::= FLOAT */ - { 163, -1 }, /* (62) type_name ::= DOUBLE */ - { 163, -4 }, /* (63) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 163, -1 }, /* (64) type_name ::= TIMESTAMP */ - { 163, -4 }, /* (65) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 163, -2 }, /* (66) type_name ::= TINYINT UNSIGNED */ - { 163, -2 }, /* (67) type_name ::= SMALLINT UNSIGNED */ - { 163, -2 }, /* (68) type_name ::= INT UNSIGNED */ - { 163, -2 }, /* (69) type_name ::= BIGINT UNSIGNED */ - { 163, -1 }, /* (70) type_name ::= JSON */ - { 163, -4 }, /* (71) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 163, -1 }, /* (72) type_name ::= MEDIUMBLOB */ - { 163, -1 }, /* (73) type_name ::= BLOB */ - { 163, -4 }, /* (74) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 163, -1 }, /* (75) type_name ::= DECIMAL */ - { 163, -4 }, /* (76) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 163, -6 }, /* (77) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 150, 0 }, /* (78) tags_def_opt ::= */ - { 150, -1 }, /* (79) tags_def_opt ::= tags_def */ - { 153, -4 }, /* (80) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 151, 0 }, /* (81) table_options ::= */ - { 151, -3 }, /* (82) table_options ::= table_options COMMENT NK_STRING */ - { 151, -3 }, /* (83) table_options ::= table_options KEEP NK_INTEGER */ - { 151, -3 }, /* (84) table_options ::= table_options TTL NK_INTEGER */ - { 151, -5 }, /* (85) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 159, -1 }, /* (86) col_name_list ::= col_name */ - { 159, -3 }, /* (87) col_name_list ::= col_name_list NK_COMMA col_name */ - { 164, -1 }, /* (88) col_name ::= column_name */ - { 140, -2 }, /* (89) cmd ::= SHOW DNODES */ - { 140, -2 }, /* (90) cmd ::= SHOW USERS */ - { 140, -2 }, /* (91) cmd ::= SHOW DATABASES */ - { 140, -4 }, /* (92) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 140, -4 }, /* (93) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 140, -3 }, /* (94) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 140, -2 }, /* (95) cmd ::= SHOW MNODES */ - { 140, -2 }, /* (96) cmd ::= SHOW MODULES */ - { 140, -2 }, /* (97) cmd ::= SHOW QNODES */ - { 140, -2 }, /* (98) cmd ::= SHOW FUNCTIONS */ - { 140, -5 }, /* (99) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 140, -2 }, /* (100) cmd ::= SHOW STREAMS */ - { 165, 0 }, /* (101) db_name_cond_opt ::= */ - { 165, -2 }, /* (102) db_name_cond_opt ::= db_name NK_DOT */ - { 166, 0 }, /* (103) like_pattern_opt ::= */ - { 166, -2 }, /* (104) like_pattern_opt ::= LIKE NK_STRING */ - { 167, -1 }, /* (105) table_name_cond ::= table_name */ - { 168, 0 }, /* (106) from_db_opt ::= */ - { 168, -2 }, /* (107) from_db_opt ::= FROM db_name */ - { 140, -1 }, /* (108) cmd ::= query_expression */ - { 170, -1 }, /* (109) literal ::= NK_INTEGER */ - { 170, -1 }, /* (110) literal ::= NK_FLOAT */ - { 170, -1 }, /* (111) literal ::= NK_STRING */ - { 170, -1 }, /* (112) literal ::= NK_BOOL */ - { 170, -2 }, /* (113) literal ::= TIMESTAMP NK_STRING */ - { 170, -1 }, /* (114) literal ::= duration_literal */ - { 171, -1 }, /* (115) duration_literal ::= NK_VARIABLE */ - { 157, -1 }, /* (116) literal_list ::= literal */ - { 157, -3 }, /* (117) literal_list ::= literal_list NK_COMMA literal */ - { 145, -1 }, /* (118) db_name ::= NK_ID */ - { 160, -1 }, /* (119) table_name ::= NK_ID */ - { 162, -1 }, /* (120) column_name ::= NK_ID */ - { 172, -1 }, /* (121) function_name ::= NK_ID */ - { 173, -1 }, /* (122) table_alias ::= NK_ID */ - { 174, -1 }, /* (123) column_alias ::= NK_ID */ - { 141, -1 }, /* (124) user_name ::= NK_ID */ - { 175, -1 }, /* (125) expression ::= literal */ - { 175, -1 }, /* (126) expression ::= column_reference */ - { 175, -4 }, /* (127) expression ::= function_name NK_LP expression_list NK_RP */ - { 175, -4 }, /* (128) expression ::= function_name NK_LP NK_STAR NK_RP */ - { 175, -1 }, /* (129) expression ::= subquery */ - { 175, -3 }, /* (130) expression ::= NK_LP expression NK_RP */ - { 175, -2 }, /* (131) expression ::= NK_PLUS expression */ - { 175, -2 }, /* (132) expression ::= NK_MINUS expression */ - { 175, -3 }, /* (133) expression ::= expression NK_PLUS expression */ - { 175, -3 }, /* (134) expression ::= expression NK_MINUS expression */ - { 175, -3 }, /* (135) expression ::= expression NK_STAR expression */ - { 175, -3 }, /* (136) expression ::= expression NK_SLASH expression */ - { 175, -3 }, /* (137) expression ::= expression NK_REM expression */ - { 177, -1 }, /* (138) expression_list ::= expression */ - { 177, -3 }, /* (139) expression_list ::= expression_list NK_COMMA expression */ - { 176, -1 }, /* (140) column_reference ::= column_name */ - { 176, -3 }, /* (141) column_reference ::= table_name NK_DOT column_name */ - { 179, -3 }, /* (142) predicate ::= expression compare_op expression */ - { 179, -5 }, /* (143) predicate ::= expression BETWEEN expression AND expression */ - { 179, -6 }, /* (144) predicate ::= expression NOT BETWEEN expression AND expression */ - { 179, -3 }, /* (145) predicate ::= expression IS NULL */ - { 179, -4 }, /* (146) predicate ::= expression IS NOT NULL */ - { 179, -3 }, /* (147) predicate ::= expression in_op in_predicate_value */ - { 180, -1 }, /* (148) compare_op ::= NK_LT */ - { 180, -1 }, /* (149) compare_op ::= NK_GT */ - { 180, -1 }, /* (150) compare_op ::= NK_LE */ - { 180, -1 }, /* (151) compare_op ::= NK_GE */ - { 180, -1 }, /* (152) compare_op ::= NK_NE */ - { 180, -1 }, /* (153) compare_op ::= NK_EQ */ - { 180, -1 }, /* (154) compare_op ::= LIKE */ - { 180, -2 }, /* (155) compare_op ::= NOT LIKE */ - { 180, -1 }, /* (156) compare_op ::= MATCH */ - { 180, -1 }, /* (157) compare_op ::= NMATCH */ - { 181, -1 }, /* (158) in_op ::= IN */ - { 181, -2 }, /* (159) in_op ::= NOT IN */ - { 182, -3 }, /* (160) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 183, -1 }, /* (161) boolean_value_expression ::= boolean_primary */ - { 183, -2 }, /* (162) boolean_value_expression ::= NOT boolean_primary */ - { 183, -3 }, /* (163) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 183, -3 }, /* (164) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 184, -1 }, /* (165) boolean_primary ::= predicate */ - { 184, -3 }, /* (166) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 185, -1 }, /* (167) common_expression ::= expression */ - { 185, -1 }, /* (168) common_expression ::= boolean_value_expression */ - { 186, -2 }, /* (169) from_clause ::= FROM table_reference_list */ - { 187, -1 }, /* (170) table_reference_list ::= table_reference */ - { 187, -3 }, /* (171) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 188, -1 }, /* (172) table_reference ::= table_primary */ - { 188, -1 }, /* (173) table_reference ::= joined_table */ - { 189, -2 }, /* (174) table_primary ::= table_name alias_opt */ - { 189, -4 }, /* (175) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 189, -2 }, /* (176) table_primary ::= subquery alias_opt */ - { 189, -1 }, /* (177) table_primary ::= parenthesized_joined_table */ - { 191, 0 }, /* (178) alias_opt ::= */ - { 191, -1 }, /* (179) alias_opt ::= table_alias */ - { 191, -2 }, /* (180) alias_opt ::= AS table_alias */ - { 192, -3 }, /* (181) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 192, -3 }, /* (182) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 190, -6 }, /* (183) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 193, 0 }, /* (184) join_type ::= */ - { 193, -1 }, /* (185) join_type ::= INNER */ - { 195, -9 }, /* (186) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 196, 0 }, /* (187) set_quantifier_opt ::= */ - { 196, -1 }, /* (188) set_quantifier_opt ::= DISTINCT */ - { 196, -1 }, /* (189) set_quantifier_opt ::= ALL */ - { 197, -1 }, /* (190) select_list ::= NK_STAR */ - { 197, -1 }, /* (191) select_list ::= select_sublist */ - { 203, -1 }, /* (192) select_sublist ::= select_item */ - { 203, -3 }, /* (193) select_sublist ::= select_sublist NK_COMMA select_item */ - { 204, -1 }, /* (194) select_item ::= common_expression */ - { 204, -2 }, /* (195) select_item ::= common_expression column_alias */ - { 204, -3 }, /* (196) select_item ::= common_expression AS column_alias */ - { 204, -3 }, /* (197) select_item ::= table_name NK_DOT NK_STAR */ - { 198, 0 }, /* (198) where_clause_opt ::= */ - { 198, -2 }, /* (199) where_clause_opt ::= WHERE search_condition */ - { 199, 0 }, /* (200) partition_by_clause_opt ::= */ - { 199, -3 }, /* (201) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 200, 0 }, /* (202) twindow_clause_opt ::= */ - { 200, -6 }, /* (203) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ - { 200, -4 }, /* (204) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ - { 200, -6 }, /* (205) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 200, -8 }, /* (206) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 205, 0 }, /* (207) sliding_opt ::= */ - { 205, -4 }, /* (208) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 206, 0 }, /* (209) fill_opt ::= */ - { 206, -4 }, /* (210) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 206, -6 }, /* (211) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 207, -1 }, /* (212) fill_mode ::= NONE */ - { 207, -1 }, /* (213) fill_mode ::= PREV */ - { 207, -1 }, /* (214) fill_mode ::= NULL */ - { 207, -1 }, /* (215) fill_mode ::= LINEAR */ - { 207, -1 }, /* (216) fill_mode ::= NEXT */ - { 201, 0 }, /* (217) group_by_clause_opt ::= */ - { 201, -3 }, /* (218) group_by_clause_opt ::= GROUP BY group_by_list */ - { 208, -1 }, /* (219) group_by_list ::= expression */ - { 208, -3 }, /* (220) group_by_list ::= group_by_list NK_COMMA expression */ - { 202, 0 }, /* (221) having_clause_opt ::= */ - { 202, -2 }, /* (222) having_clause_opt ::= HAVING search_condition */ - { 169, -4 }, /* (223) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 209, -1 }, /* (224) query_expression_body ::= query_primary */ - { 209, -4 }, /* (225) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 213, -1 }, /* (226) query_primary ::= query_specification */ - { 210, 0 }, /* (227) order_by_clause_opt ::= */ - { 210, -3 }, /* (228) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 211, 0 }, /* (229) slimit_clause_opt ::= */ - { 211, -2 }, /* (230) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 211, -4 }, /* (231) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 211, -4 }, /* (232) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 212, 0 }, /* (233) limit_clause_opt ::= */ - { 212, -2 }, /* (234) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 212, -4 }, /* (235) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 212, -4 }, /* (236) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 178, -3 }, /* (237) subquery ::= NK_LP query_expression NK_RP */ - { 194, -1 }, /* (238) search_condition ::= common_expression */ - { 214, -1 }, /* (239) sort_specification_list ::= sort_specification */ - { 214, -3 }, /* (240) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 215, -3 }, /* (241) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 216, 0 }, /* (242) ordering_specification_opt ::= */ - { 216, -1 }, /* (243) ordering_specification_opt ::= ASC */ - { 216, -1 }, /* (244) ordering_specification_opt ::= DESC */ - { 217, 0 }, /* (245) null_ordering_opt ::= */ - { 217, -2 }, /* (246) null_ordering_opt ::= NULLS FIRST */ - { 217, -2 }, /* (247) null_ordering_opt ::= NULLS LAST */ + { 163, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 163, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 164, 0 }, /* (2) account_options ::= */ + { 164, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 164, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 164, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 164, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 164, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 164, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 164, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 164, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 164, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 165, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 165, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 167, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 167, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 167, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 167, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 167, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 167, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 167, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 167, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 167, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 167, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 163, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ + { 163, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 163, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ + { 163, -3 }, /* (27) cmd ::= DROP USER user_name */ + { 163, -3 }, /* (28) cmd ::= CREATE DNODE dnode_endpoint */ + { 163, -5 }, /* (29) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ + { 163, -3 }, /* (30) cmd ::= DROP DNODE NK_INTEGER */ + { 163, -3 }, /* (31) cmd ::= DROP DNODE dnode_endpoint */ + { 163, -4 }, /* (32) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 163, -5 }, /* (33) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 163, -4 }, /* (34) cmd ::= ALTER ALL DNODES NK_STRING */ + { 163, -5 }, /* (35) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 169, -1 }, /* (36) dnode_endpoint ::= NK_STRING */ + { 170, -1 }, /* (37) dnode_host_name ::= NK_ID */ + { 170, -1 }, /* (38) dnode_host_name ::= NK_IPTOKEN */ + { 163, -3 }, /* (39) cmd ::= ALTER LOCAL NK_STRING */ + { 163, -4 }, /* (40) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 163, -5 }, /* (41) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 163, -5 }, /* (42) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 163, -5 }, /* (43) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 163, -4 }, /* (44) cmd ::= DROP DATABASE exists_opt db_name */ + { 163, -2 }, /* (45) cmd ::= USE db_name */ + { 163, -4 }, /* (46) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 171, -3 }, /* (47) not_exists_opt ::= IF NOT EXISTS */ + { 171, 0 }, /* (48) not_exists_opt ::= */ + { 174, -2 }, /* (49) exists_opt ::= IF EXISTS */ + { 174, 0 }, /* (50) exists_opt ::= */ + { 173, 0 }, /* (51) db_options ::= */ + { 173, -3 }, /* (52) db_options ::= db_options BLOCKS NK_INTEGER */ + { 173, -3 }, /* (53) db_options ::= db_options CACHE NK_INTEGER */ + { 173, -3 }, /* (54) db_options ::= db_options CACHELAST NK_INTEGER */ + { 173, -3 }, /* (55) db_options ::= db_options COMP NK_INTEGER */ + { 173, -3 }, /* (56) db_options ::= db_options DAYS NK_INTEGER */ + { 173, -3 }, /* (57) db_options ::= db_options FSYNC NK_INTEGER */ + { 173, -3 }, /* (58) db_options ::= db_options MAXROWS NK_INTEGER */ + { 173, -3 }, /* (59) db_options ::= db_options MINROWS NK_INTEGER */ + { 173, -3 }, /* (60) db_options ::= db_options KEEP NK_INTEGER */ + { 173, -3 }, /* (61) db_options ::= db_options PRECISION NK_STRING */ + { 173, -3 }, /* (62) db_options ::= db_options QUORUM NK_INTEGER */ + { 173, -3 }, /* (63) db_options ::= db_options REPLICA NK_INTEGER */ + { 173, -3 }, /* (64) db_options ::= db_options TTL NK_INTEGER */ + { 173, -3 }, /* (65) db_options ::= db_options WAL NK_INTEGER */ + { 173, -3 }, /* (66) db_options ::= db_options VGROUPS NK_INTEGER */ + { 173, -3 }, /* (67) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 173, -3 }, /* (68) db_options ::= db_options STREAM_MODE NK_INTEGER */ + { 173, -3 }, /* (69) db_options ::= db_options RETENTIONS NK_STRING */ + { 173, -3 }, /* (70) db_options ::= db_options FILE_FACTOR NK_FLOAT */ + { 175, -1 }, /* (71) alter_db_options ::= alter_db_option */ + { 175, -2 }, /* (72) alter_db_options ::= alter_db_options alter_db_option */ + { 176, -2 }, /* (73) alter_db_option ::= BLOCKS NK_INTEGER */ + { 176, -2 }, /* (74) alter_db_option ::= FSYNC NK_INTEGER */ + { 176, -2 }, /* (75) alter_db_option ::= KEEP NK_INTEGER */ + { 176, -2 }, /* (76) alter_db_option ::= WAL NK_INTEGER */ + { 176, -2 }, /* (77) alter_db_option ::= QUORUM NK_INTEGER */ + { 176, -2 }, /* (78) alter_db_option ::= CACHELAST NK_INTEGER */ + { 163, -9 }, /* (79) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 163, -3 }, /* (80) cmd ::= CREATE TABLE multi_create_clause */ + { 163, -9 }, /* (81) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 163, -3 }, /* (82) cmd ::= DROP TABLE multi_drop_clause */ + { 163, -4 }, /* (83) cmd ::= DROP STABLE exists_opt full_table_name */ + { 163, -3 }, /* (84) cmd ::= ALTER TABLE alter_table_clause */ + { 163, -3 }, /* (85) cmd ::= ALTER STABLE alter_table_clause */ + { 184, -2 }, /* (86) alter_table_clause ::= full_table_name alter_table_options */ + { 184, -5 }, /* (87) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 184, -4 }, /* (88) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 184, -5 }, /* (89) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 184, -5 }, /* (90) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 184, -5 }, /* (91) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 184, -4 }, /* (92) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 184, -5 }, /* (93) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 184, -5 }, /* (94) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 184, -6 }, /* (95) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ + { 181, -1 }, /* (96) multi_create_clause ::= create_subtable_clause */ + { 181, -2 }, /* (97) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 188, -9 }, /* (98) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ + { 183, -1 }, /* (99) multi_drop_clause ::= drop_table_clause */ + { 183, -2 }, /* (100) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 191, -2 }, /* (101) drop_table_clause ::= exists_opt full_table_name */ + { 189, 0 }, /* (102) specific_tags_opt ::= */ + { 189, -3 }, /* (103) specific_tags_opt ::= NK_LP col_name_list NK_RP */ + { 177, -1 }, /* (104) full_table_name ::= table_name */ + { 177, -3 }, /* (105) full_table_name ::= db_name NK_DOT table_name */ + { 178, -1 }, /* (106) column_def_list ::= column_def */ + { 178, -3 }, /* (107) column_def_list ::= column_def_list NK_COMMA column_def */ + { 194, -2 }, /* (108) column_def ::= column_name type_name */ + { 194, -4 }, /* (109) column_def ::= column_name type_name COMMENT NK_STRING */ + { 187, -1 }, /* (110) type_name ::= BOOL */ + { 187, -1 }, /* (111) type_name ::= TINYINT */ + { 187, -1 }, /* (112) type_name ::= SMALLINT */ + { 187, -1 }, /* (113) type_name ::= INT */ + { 187, -1 }, /* (114) type_name ::= INTEGER */ + { 187, -1 }, /* (115) type_name ::= BIGINT */ + { 187, -1 }, /* (116) type_name ::= FLOAT */ + { 187, -1 }, /* (117) type_name ::= DOUBLE */ + { 187, -4 }, /* (118) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 187, -1 }, /* (119) type_name ::= TIMESTAMP */ + { 187, -4 }, /* (120) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 187, -2 }, /* (121) type_name ::= TINYINT UNSIGNED */ + { 187, -2 }, /* (122) type_name ::= SMALLINT UNSIGNED */ + { 187, -2 }, /* (123) type_name ::= INT UNSIGNED */ + { 187, -2 }, /* (124) type_name ::= BIGINT UNSIGNED */ + { 187, -1 }, /* (125) type_name ::= JSON */ + { 187, -4 }, /* (126) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 187, -1 }, /* (127) type_name ::= MEDIUMBLOB */ + { 187, -1 }, /* (128) type_name ::= BLOB */ + { 187, -4 }, /* (129) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 187, -1 }, /* (130) type_name ::= DECIMAL */ + { 187, -4 }, /* (131) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 187, -6 }, /* (132) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 179, 0 }, /* (133) tags_def_opt ::= */ + { 179, -1 }, /* (134) tags_def_opt ::= tags_def */ + { 182, -4 }, /* (135) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 180, 0 }, /* (136) table_options ::= */ + { 180, -3 }, /* (137) table_options ::= table_options COMMENT NK_STRING */ + { 180, -3 }, /* (138) table_options ::= table_options KEEP NK_INTEGER */ + { 180, -3 }, /* (139) table_options ::= table_options TTL NK_INTEGER */ + { 180, -5 }, /* (140) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 180, -5 }, /* (141) table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ + { 185, -1 }, /* (142) alter_table_options ::= alter_table_option */ + { 185, -2 }, /* (143) alter_table_options ::= alter_table_options alter_table_option */ + { 196, -2 }, /* (144) alter_table_option ::= COMMENT NK_STRING */ + { 196, -2 }, /* (145) alter_table_option ::= KEEP NK_INTEGER */ + { 196, -2 }, /* (146) alter_table_option ::= TTL NK_INTEGER */ + { 192, -1 }, /* (147) col_name_list ::= col_name */ + { 192, -3 }, /* (148) col_name_list ::= col_name_list NK_COMMA col_name */ + { 197, -1 }, /* (149) col_name ::= column_name */ + { 163, -2 }, /* (150) cmd ::= SHOW DNODES */ + { 163, -2 }, /* (151) cmd ::= SHOW USERS */ + { 163, -2 }, /* (152) cmd ::= SHOW DATABASES */ + { 163, -4 }, /* (153) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 163, -4 }, /* (154) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 163, -3 }, /* (155) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 163, -2 }, /* (156) cmd ::= SHOW MNODES */ + { 163, -2 }, /* (157) cmd ::= SHOW MODULES */ + { 163, -2 }, /* (158) cmd ::= SHOW QNODES */ + { 163, -2 }, /* (159) cmd ::= SHOW FUNCTIONS */ + { 163, -5 }, /* (160) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 163, -2 }, /* (161) cmd ::= SHOW STREAMS */ + { 198, 0 }, /* (162) db_name_cond_opt ::= */ + { 198, -2 }, /* (163) db_name_cond_opt ::= db_name NK_DOT */ + { 199, 0 }, /* (164) like_pattern_opt ::= */ + { 199, -2 }, /* (165) like_pattern_opt ::= LIKE NK_STRING */ + { 200, -1 }, /* (166) table_name_cond ::= table_name */ + { 201, 0 }, /* (167) from_db_opt ::= */ + { 201, -2 }, /* (168) from_db_opt ::= FROM db_name */ + { 195, -1 }, /* (169) func_name_list ::= func_name */ + { 195, -3 }, /* (170) func_name_list ::= func_name_list NK_COMMA col_name */ + { 202, -1 }, /* (171) func_name ::= function_name */ + { 163, -7 }, /* (172) cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ + { 163, -9 }, /* (173) cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ + { 163, -5 }, /* (174) cmd ::= DROP INDEX index_name ON table_name */ + { 205, 0 }, /* (175) index_options ::= */ + { 205, -9 }, /* (176) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ + { 205, -11 }, /* (177) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ + { 206, -1 }, /* (178) func_list ::= func */ + { 206, -3 }, /* (179) func_list ::= func_list NK_COMMA func */ + { 209, -4 }, /* (180) func ::= function_name NK_LP expression_list NK_RP */ + { 163, -6 }, /* (181) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ + { 163, -6 }, /* (182) cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ + { 163, -4 }, /* (183) cmd ::= DROP TOPIC exists_opt topic_name */ + { 163, -1 }, /* (184) cmd ::= query_expression */ + { 166, -1 }, /* (185) literal ::= NK_INTEGER */ + { 166, -1 }, /* (186) literal ::= NK_FLOAT */ + { 166, -1 }, /* (187) literal ::= NK_STRING */ + { 166, -1 }, /* (188) literal ::= NK_BOOL */ + { 166, -2 }, /* (189) literal ::= TIMESTAMP NK_STRING */ + { 166, -1 }, /* (190) literal ::= duration_literal */ + { 207, -1 }, /* (191) duration_literal ::= NK_VARIABLE */ + { 190, -1 }, /* (192) literal_list ::= literal */ + { 190, -3 }, /* (193) literal_list ::= literal_list NK_COMMA literal */ + { 172, -1 }, /* (194) db_name ::= NK_ID */ + { 193, -1 }, /* (195) table_name ::= NK_ID */ + { 186, -1 }, /* (196) column_name ::= NK_ID */ + { 203, -1 }, /* (197) function_name ::= NK_ID */ + { 213, -1 }, /* (198) table_alias ::= NK_ID */ + { 214, -1 }, /* (199) column_alias ::= NK_ID */ + { 168, -1 }, /* (200) user_name ::= NK_ID */ + { 204, -1 }, /* (201) index_name ::= NK_ID */ + { 211, -1 }, /* (202) topic_name ::= NK_ID */ + { 215, -1 }, /* (203) expression ::= literal */ + { 215, -1 }, /* (204) expression ::= column_reference */ + { 215, -4 }, /* (205) expression ::= function_name NK_LP expression_list NK_RP */ + { 215, -4 }, /* (206) expression ::= function_name NK_LP NK_STAR NK_RP */ + { 215, -1 }, /* (207) expression ::= subquery */ + { 215, -3 }, /* (208) expression ::= NK_LP expression NK_RP */ + { 215, -2 }, /* (209) expression ::= NK_PLUS expression */ + { 215, -2 }, /* (210) expression ::= NK_MINUS expression */ + { 215, -3 }, /* (211) expression ::= expression NK_PLUS expression */ + { 215, -3 }, /* (212) expression ::= expression NK_MINUS expression */ + { 215, -3 }, /* (213) expression ::= expression NK_STAR expression */ + { 215, -3 }, /* (214) expression ::= expression NK_SLASH expression */ + { 215, -3 }, /* (215) expression ::= expression NK_REM expression */ + { 210, -1 }, /* (216) expression_list ::= expression */ + { 210, -3 }, /* (217) expression_list ::= expression_list NK_COMMA expression */ + { 216, -1 }, /* (218) column_reference ::= column_name */ + { 216, -3 }, /* (219) column_reference ::= table_name NK_DOT column_name */ + { 218, -3 }, /* (220) predicate ::= expression compare_op expression */ + { 218, -5 }, /* (221) predicate ::= expression BETWEEN expression AND expression */ + { 218, -6 }, /* (222) predicate ::= expression NOT BETWEEN expression AND expression */ + { 218, -3 }, /* (223) predicate ::= expression IS NULL */ + { 218, -4 }, /* (224) predicate ::= expression IS NOT NULL */ + { 218, -3 }, /* (225) predicate ::= expression in_op in_predicate_value */ + { 219, -1 }, /* (226) compare_op ::= NK_LT */ + { 219, -1 }, /* (227) compare_op ::= NK_GT */ + { 219, -1 }, /* (228) compare_op ::= NK_LE */ + { 219, -1 }, /* (229) compare_op ::= NK_GE */ + { 219, -1 }, /* (230) compare_op ::= NK_NE */ + { 219, -1 }, /* (231) compare_op ::= NK_EQ */ + { 219, -1 }, /* (232) compare_op ::= LIKE */ + { 219, -2 }, /* (233) compare_op ::= NOT LIKE */ + { 219, -1 }, /* (234) compare_op ::= MATCH */ + { 219, -1 }, /* (235) compare_op ::= NMATCH */ + { 220, -1 }, /* (236) in_op ::= IN */ + { 220, -2 }, /* (237) in_op ::= NOT IN */ + { 221, -3 }, /* (238) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 222, -1 }, /* (239) boolean_value_expression ::= boolean_primary */ + { 222, -2 }, /* (240) boolean_value_expression ::= NOT boolean_primary */ + { 222, -3 }, /* (241) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 222, -3 }, /* (242) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 223, -1 }, /* (243) boolean_primary ::= predicate */ + { 223, -3 }, /* (244) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 224, -1 }, /* (245) common_expression ::= expression */ + { 224, -1 }, /* (246) common_expression ::= boolean_value_expression */ + { 225, -2 }, /* (247) from_clause ::= FROM table_reference_list */ + { 226, -1 }, /* (248) table_reference_list ::= table_reference */ + { 226, -3 }, /* (249) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 227, -1 }, /* (250) table_reference ::= table_primary */ + { 227, -1 }, /* (251) table_reference ::= joined_table */ + { 228, -2 }, /* (252) table_primary ::= table_name alias_opt */ + { 228, -4 }, /* (253) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 228, -2 }, /* (254) table_primary ::= subquery alias_opt */ + { 228, -1 }, /* (255) table_primary ::= parenthesized_joined_table */ + { 230, 0 }, /* (256) alias_opt ::= */ + { 230, -1 }, /* (257) alias_opt ::= table_alias */ + { 230, -2 }, /* (258) alias_opt ::= AS table_alias */ + { 231, -3 }, /* (259) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 231, -3 }, /* (260) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 229, -6 }, /* (261) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 232, 0 }, /* (262) join_type ::= */ + { 232, -1 }, /* (263) join_type ::= INNER */ + { 234, -9 }, /* (264) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 235, 0 }, /* (265) set_quantifier_opt ::= */ + { 235, -1 }, /* (266) set_quantifier_opt ::= DISTINCT */ + { 235, -1 }, /* (267) set_quantifier_opt ::= ALL */ + { 236, -1 }, /* (268) select_list ::= NK_STAR */ + { 236, -1 }, /* (269) select_list ::= select_sublist */ + { 242, -1 }, /* (270) select_sublist ::= select_item */ + { 242, -3 }, /* (271) select_sublist ::= select_sublist NK_COMMA select_item */ + { 243, -1 }, /* (272) select_item ::= common_expression */ + { 243, -2 }, /* (273) select_item ::= common_expression column_alias */ + { 243, -3 }, /* (274) select_item ::= common_expression AS column_alias */ + { 243, -3 }, /* (275) select_item ::= table_name NK_DOT NK_STAR */ + { 237, 0 }, /* (276) where_clause_opt ::= */ + { 237, -2 }, /* (277) where_clause_opt ::= WHERE search_condition */ + { 238, 0 }, /* (278) partition_by_clause_opt ::= */ + { 238, -3 }, /* (279) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 239, 0 }, /* (280) twindow_clause_opt ::= */ + { 239, -6 }, /* (281) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 239, -4 }, /* (282) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 239, -6 }, /* (283) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 239, -8 }, /* (284) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 208, 0 }, /* (285) sliding_opt ::= */ + { 208, -4 }, /* (286) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 244, 0 }, /* (287) fill_opt ::= */ + { 244, -4 }, /* (288) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 244, -6 }, /* (289) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 245, -1 }, /* (290) fill_mode ::= NONE */ + { 245, -1 }, /* (291) fill_mode ::= PREV */ + { 245, -1 }, /* (292) fill_mode ::= NULL */ + { 245, -1 }, /* (293) fill_mode ::= LINEAR */ + { 245, -1 }, /* (294) fill_mode ::= NEXT */ + { 240, 0 }, /* (295) group_by_clause_opt ::= */ + { 240, -3 }, /* (296) group_by_clause_opt ::= GROUP BY group_by_list */ + { 246, -1 }, /* (297) group_by_list ::= expression */ + { 246, -3 }, /* (298) group_by_list ::= group_by_list NK_COMMA expression */ + { 241, 0 }, /* (299) having_clause_opt ::= */ + { 241, -2 }, /* (300) having_clause_opt ::= HAVING search_condition */ + { 212, -4 }, /* (301) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 247, -1 }, /* (302) query_expression_body ::= query_primary */ + { 247, -4 }, /* (303) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 251, -1 }, /* (304) query_primary ::= query_specification */ + { 248, 0 }, /* (305) order_by_clause_opt ::= */ + { 248, -3 }, /* (306) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 249, 0 }, /* (307) slimit_clause_opt ::= */ + { 249, -2 }, /* (308) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 249, -4 }, /* (309) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 249, -4 }, /* (310) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 250, 0 }, /* (311) limit_clause_opt ::= */ + { 250, -2 }, /* (312) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 250, -4 }, /* (313) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 250, -4 }, /* (314) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 217, -3 }, /* (315) subquery ::= NK_LP query_expression NK_RP */ + { 233, -1 }, /* (316) search_condition ::= common_expression */ + { 252, -1 }, /* (317) sort_specification_list ::= sort_specification */ + { 252, -3 }, /* (318) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 253, -3 }, /* (319) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 254, 0 }, /* (320) ordering_specification_opt ::= */ + { 254, -1 }, /* (321) ordering_specification_opt ::= ASC */ + { 254, -1 }, /* (322) ordering_specification_opt ::= DESC */ + { 255, 0 }, /* (323) null_ordering_opt ::= */ + { 255, -2 }, /* (324) null_ordering_opt ::= NULLS FIRST */ + { 255, -2 }, /* (325) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1953,780 +2257,992 @@ static YYACTIONTYPE yy_reduce( */ /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; - case 0: /* cmd ::= CREATE USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy5, &yymsp[0].minor.yy0);} + case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ +{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } + yy_destructor(yypParser,164,&yymsp[0].minor); + break; + case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ +{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } + yy_destructor(yypParser,165,&yymsp[0].minor); + break; + case 2: /* account_options ::= */ +{ } + break; + case 3: /* account_options ::= account_options PPS literal */ + case 4: /* account_options ::= account_options TSERIES literal */ yytestcase(yyruleno==4); + case 5: /* account_options ::= account_options STORAGE literal */ yytestcase(yyruleno==5); + case 6: /* account_options ::= account_options STREAMS literal */ yytestcase(yyruleno==6); + case 7: /* account_options ::= account_options QTIME literal */ yytestcase(yyruleno==7); + case 8: /* account_options ::= account_options DBS literal */ yytestcase(yyruleno==8); + case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); + case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); + case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); +{ yy_destructor(yypParser,164,&yymsp[-2].minor); +{ } + yy_destructor(yypParser,166,&yymsp[0].minor); +} + break; + case 12: /* alter_account_options ::= alter_account_option */ +{ yy_destructor(yypParser,167,&yymsp[0].minor); +{ } +} + break; + case 13: /* alter_account_options ::= alter_account_options alter_account_option */ +{ yy_destructor(yypParser,165,&yymsp[-1].minor); +{ } + yy_destructor(yypParser,167,&yymsp[0].minor); +} + break; + case 14: /* alter_account_option ::= PASS literal */ + case 15: /* alter_account_option ::= PPS literal */ yytestcase(yyruleno==15); + case 16: /* alter_account_option ::= TSERIES literal */ yytestcase(yyruleno==16); + case 17: /* alter_account_option ::= STORAGE literal */ yytestcase(yyruleno==17); + case 18: /* alter_account_option ::= STREAMS literal */ yytestcase(yyruleno==18); + case 19: /* alter_account_option ::= QTIME literal */ yytestcase(yyruleno==19); + case 20: /* alter_account_option ::= DBS literal */ yytestcase(yyruleno==20); + case 21: /* alter_account_option ::= USERS literal */ yytestcase(yyruleno==21); + case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); + case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); +{ } + yy_destructor(yypParser,166,&yymsp[0].minor); + break; + case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING */ +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy417, &yymsp[0].minor.yy0); } + break; + case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy417, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } + break; + case 26: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy417, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } + break; + case 27: /* cmd ::= DROP USER user_name */ +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy417); } + break; + case 28: /* cmd ::= CREATE DNODE dnode_endpoint */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy417, NULL); } + break; + case 29: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy417, &yymsp[0].minor.yy0); } + break; + case 30: /* cmd ::= DROP DNODE NK_INTEGER */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } + break; + case 31: /* cmd ::= DROP DNODE dnode_endpoint */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy417); } + break; + case 32: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ +{ pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } + break; + case 33: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ +{ pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 34: /* cmd ::= ALTER ALL DNODES NK_STRING */ +{ pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } + break; + case 35: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ +{ pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 36: /* dnode_endpoint ::= NK_STRING */ + case 37: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==37); + case 38: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==38); + case 194: /* db_name ::= NK_ID */ yytestcase(yyruleno==194); + case 195: /* table_name ::= NK_ID */ yytestcase(yyruleno==195); + case 196: /* column_name ::= NK_ID */ yytestcase(yyruleno==196); + case 197: /* function_name ::= NK_ID */ yytestcase(yyruleno==197); + case 198: /* table_alias ::= NK_ID */ yytestcase(yyruleno==198); + case 199: /* column_alias ::= NK_ID */ yytestcase(yyruleno==199); + case 200: /* user_name ::= NK_ID */ yytestcase(yyruleno==200); + case 201: /* index_name ::= NK_ID */ yytestcase(yyruleno==201); + case 202: /* topic_name ::= NK_ID */ yytestcase(yyruleno==202); +{ yylhsminor.yy417 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy417 = yylhsminor.yy417; + break; + case 39: /* cmd ::= ALTER LOCAL NK_STRING */ +{ pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } + break; + case 40: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ +{ pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 41: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ +{ pCxt->pRootNode = createCreateQnodeStmt(pCxt, &yymsp[0].minor.yy0); } + break; + case 42: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ +{ pCxt->pRootNode = createDropQnodeStmt(pCxt, &yymsp[0].minor.yy0); } + break; + case 43: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy345, &yymsp[-1].minor.yy417, yymsp[0].minor.yy360); } + break; + case 44: /* cmd ::= DROP DATABASE exists_opt db_name */ +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy345, &yymsp[0].minor.yy417); } break; - case 1: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy5, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0);} + case 45: /* cmd ::= USE db_name */ +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy417); } break; - case 2: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy5, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0);} + case 46: /* cmd ::= ALTER DATABASE db_name alter_db_options */ +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy417, yymsp[0].minor.yy360); } break; - case 3: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy5); } + case 47: /* not_exists_opt ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy345 = true; } break; - case 4: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy5, NULL);} + case 48: /* not_exists_opt ::= */ + case 50: /* exists_opt ::= */ yytestcase(yyruleno==50); + case 265: /* set_quantifier_opt ::= */ yytestcase(yyruleno==265); +{ yymsp[1].minor.yy345 = false; } break; - case 5: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy5, &yymsp[0].minor.yy0);} + case 49: /* exists_opt ::= IF EXISTS */ +{ yymsp[-1].minor.yy345 = true; } break; - case 6: /* cmd ::= DROP DNODE NK_INTEGER */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0);} + case 51: /* db_options ::= */ +{ yymsp[1].minor.yy360 = createDefaultDatabaseOptions(pCxt); } break; - case 7: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy5);} + case 52: /* db_options ::= db_options BLOCKS NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_BLOCKS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 8: /* dnode_endpoint ::= NK_STRING */ - case 9: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==9); - case 10: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==10); - case 118: /* db_name ::= NK_ID */ yytestcase(yyruleno==118); - case 119: /* table_name ::= NK_ID */ yytestcase(yyruleno==119); - case 120: /* column_name ::= NK_ID */ yytestcase(yyruleno==120); - case 121: /* function_name ::= NK_ID */ yytestcase(yyruleno==121); - case 122: /* table_alias ::= NK_ID */ yytestcase(yyruleno==122); - case 123: /* column_alias ::= NK_ID */ yytestcase(yyruleno==123); - case 124: /* user_name ::= NK_ID */ yytestcase(yyruleno==124); -{ yylhsminor.yy5 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy5 = yylhsminor.yy5; + case 53: /* db_options ::= db_options CACHE NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_CACHE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 11: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy25, &yymsp[-1].minor.yy5, yymsp[0].minor.yy339);} + case 54: /* db_options ::= db_options CACHELAST NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 12: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy25, &yymsp[0].minor.yy5); } + case 55: /* db_options ::= db_options COMP NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 13: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy5);} + case 56: /* db_options ::= db_options DAYS NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 14: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy25 = true; } + case 57: /* db_options ::= db_options FSYNC NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 15: /* not_exists_opt ::= */ - case 17: /* exists_opt ::= */ yytestcase(yyruleno==17); - case 187: /* set_quantifier_opt ::= */ yytestcase(yyruleno==187); -{ yymsp[1].minor.yy25 = false; } + case 58: /* db_options ::= db_options MAXROWS NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 16: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy25 = true; } + case 59: /* db_options ::= db_options MINROWS NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 18: /* db_options ::= */ -{ yymsp[1].minor.yy339 = createDefaultDatabaseOptions(pCxt); } + case 60: /* db_options ::= db_options KEEP NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_KEEP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 19: /* db_options ::= db_options BLOCKS NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_BLOCKS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; + case 61: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 20: /* db_options ::= db_options CACHE NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_CACHE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 21: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 22: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 23: /* db_options ::= db_options DAYS NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 24: /* db_options ::= db_options FSYNC NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 25: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 26: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 27: /* db_options ::= db_options KEEP NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_KEEP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 28: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 29: /* db_options ::= db_options QUORUM NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_QUORUM, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 30: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 31: /* db_options ::= db_options TTL NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; + case 62: /* db_options ::= db_options QUORUM NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_QUORUM, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 32: /* db_options ::= db_options WAL NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 33: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; - break; - case 34: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_SINGLESTABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; + case 63: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 35: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ -{ yylhsminor.yy339 = setDatabaseOption(pCxt, yymsp[-2].minor.yy339, DB_OPTION_STREAMMODE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy339 = yylhsminor.yy339; + case 64: /* db_options ::= db_options TTL NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 36: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 38: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==38); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy25, yymsp[-5].minor.yy68, yymsp[-3].minor.yy40, yymsp[-1].minor.yy40, yymsp[0].minor.yy418);} + case 65: /* db_options ::= db_options WAL NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 37: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy40);} + case 66: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 39: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy40); } + case 67: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 40: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy25, yymsp[0].minor.yy68); } + case 68: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_STREAM_MODE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 41: /* multi_create_clause ::= create_subtable_clause */ - case 44: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==44); - case 51: /* column_def_list ::= column_def */ yytestcase(yyruleno==51); - case 86: /* col_name_list ::= col_name */ yytestcase(yyruleno==86); - case 192: /* select_sublist ::= select_item */ yytestcase(yyruleno==192); - case 239: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==239); -{ yylhsminor.yy40 = createNodeList(pCxt, yymsp[0].minor.yy68); } - yymsp[0].minor.yy40 = yylhsminor.yy40; + case 69: /* db_options ::= db_options RETENTIONS NK_STRING */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_RETENTIONS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 42: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 45: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==45); -{ yylhsminor.yy40 = addNodeToList(pCxt, yymsp[-1].minor.yy40, yymsp[0].minor.yy68); } - yymsp[-1].minor.yy40 = yylhsminor.yy40; + case 70: /* db_options ::= db_options FILE_FACTOR NK_FLOAT */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-2].minor.yy360, DB_OPTION_FILE_FACTOR, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 43: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ -{ yylhsminor.yy68 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy25, yymsp[-7].minor.yy68, yymsp[-5].minor.yy68, yymsp[-4].minor.yy40, yymsp[-1].minor.yy40); } - yymsp[-8].minor.yy68 = yylhsminor.yy68; + case 71: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy360 = createDefaultAlterDatabaseOptions(pCxt); yylhsminor.yy360 = setDatabaseOption(pCxt, yylhsminor.yy360, yymsp[0].minor.yy285.type, &yymsp[0].minor.yy285.val); } + yymsp[0].minor.yy360 = yylhsminor.yy360; break; - case 46: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy68 = createDropTableClause(pCxt, yymsp[-1].minor.yy25, yymsp[0].minor.yy68); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; + case 72: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy360 = setDatabaseOption(pCxt, yymsp[-1].minor.yy360, yymsp[0].minor.yy285.type, &yymsp[0].minor.yy285.val); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 47: /* specific_tags_opt ::= */ - case 78: /* tags_def_opt ::= */ yytestcase(yyruleno==78); - case 200: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==200); - case 217: /* group_by_clause_opt ::= */ yytestcase(yyruleno==217); - case 227: /* order_by_clause_opt ::= */ yytestcase(yyruleno==227); -{ yymsp[1].minor.yy40 = NULL; } + case 73: /* alter_db_option ::= BLOCKS NK_INTEGER */ +{ yymsp[-1].minor.yy285.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } break; - case 48: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy40 = yymsp[-1].minor.yy40; } + case 74: /* alter_db_option ::= FSYNC NK_INTEGER */ +{ yymsp[-1].minor.yy285.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } + break; + case 75: /* alter_db_option ::= KEEP NK_INTEGER */ +{ yymsp[-1].minor.yy285.type = DB_OPTION_KEEP; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } break; - case 49: /* full_table_name ::= table_name */ -{ yylhsminor.yy68 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy5, NULL); } - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 76: /* alter_db_option ::= WAL NK_INTEGER */ +{ yymsp[-1].minor.yy285.type = DB_OPTION_WAL; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } break; - case 50: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy68 = createRealTableNode(pCxt, &yymsp[-2].minor.yy5, &yymsp[0].minor.yy5, NULL); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 77: /* alter_db_option ::= QUORUM NK_INTEGER */ +{ yymsp[-1].minor.yy285.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } break; - case 52: /* column_def_list ::= column_def_list NK_COMMA column_def */ - case 87: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==87); - case 193: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==193); - case 240: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==240); -{ yylhsminor.yy40 = addNodeToList(pCxt, yymsp[-2].minor.yy40, yymsp[0].minor.yy68); } - yymsp[-2].minor.yy40 = yylhsminor.yy40; + case 78: /* alter_db_option ::= CACHELAST NK_INTEGER */ +{ yymsp[-1].minor.yy285.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } break; - case 53: /* column_def ::= column_name type_name */ -{ yylhsminor.yy68 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy5, yymsp[0].minor.yy372, NULL); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; + case 79: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 81: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==81); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy345, yymsp[-5].minor.yy360, yymsp[-3].minor.yy24, yymsp[-1].minor.yy24, yymsp[0].minor.yy360); } + break; + case 80: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy24); } + break; + case 82: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy24); } + break; + case 83: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy345, yymsp[0].minor.yy360); } + break; + case 84: /* cmd ::= ALTER TABLE alter_table_clause */ + case 85: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==85); + case 184: /* cmd ::= query_expression */ yytestcase(yyruleno==184); +{ pCxt->pRootNode = yymsp[0].minor.yy360; } + break; + case 86: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy360 = createAlterTableOption(pCxt, yymsp[-1].minor.yy360, yymsp[0].minor.yy360); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; + break; + case 87: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy360 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy360, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy417, yymsp[0].minor.yy400); } + yymsp[-4].minor.yy360 = yylhsminor.yy360; break; - case 54: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy68 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy5, yymsp[-2].minor.yy372, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; + case 88: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy360 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy360, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy417); } + yymsp[-3].minor.yy360 = yylhsminor.yy360; break; - case 55: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 89: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy360 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy360, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy417, yymsp[0].minor.yy400); } + yymsp[-4].minor.yy360 = yylhsminor.yy360; break; - case 56: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 90: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy360 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy360, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy417, &yymsp[0].minor.yy417); } + yymsp[-4].minor.yy360 = yylhsminor.yy360; break; - case 57: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 91: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy360 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy360, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy417, yymsp[0].minor.yy400); } + yymsp[-4].minor.yy360 = yylhsminor.yy360; break; - case 58: /* type_name ::= INT */ - case 59: /* type_name ::= INTEGER */ yytestcase(yyruleno==59); -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_INT); } + case 92: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy360 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy360, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy417); } + yymsp[-3].minor.yy360 = yylhsminor.yy360; break; - case 60: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 93: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy360 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy360, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy417, yymsp[0].minor.yy400); } + yymsp[-4].minor.yy360 = yylhsminor.yy360; break; - case 61: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 94: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy360 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy360, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy417, &yymsp[0].minor.yy417); } + yymsp[-4].minor.yy360 = yylhsminor.yy360; break; - case 62: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 95: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ +{ yylhsminor.yy360 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy360, &yymsp[-2].minor.yy417, yymsp[0].minor.yy360); } + yymsp[-5].minor.yy360 = yylhsminor.yy360; break; - case 63: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy372 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 96: /* multi_create_clause ::= create_subtable_clause */ + case 99: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==99); + case 106: /* column_def_list ::= column_def */ yytestcase(yyruleno==106); + case 147: /* col_name_list ::= col_name */ yytestcase(yyruleno==147); + case 169: /* func_name_list ::= func_name */ yytestcase(yyruleno==169); + case 178: /* func_list ::= func */ yytestcase(yyruleno==178); + case 270: /* select_sublist ::= select_item */ yytestcase(yyruleno==270); + case 317: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==317); +{ yylhsminor.yy24 = createNodeList(pCxt, yymsp[0].minor.yy360); } + yymsp[0].minor.yy24 = yylhsminor.yy24; break; - case 64: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 97: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 100: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==100); +{ yylhsminor.yy24 = addNodeToList(pCxt, yymsp[-1].minor.yy24, yymsp[0].minor.yy360); } + yymsp[-1].minor.yy24 = yylhsminor.yy24; break; - case 65: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy372 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 98: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ +{ yylhsminor.yy360 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy345, yymsp[-7].minor.yy360, yymsp[-5].minor.yy360, yymsp[-4].minor.yy24, yymsp[-1].minor.yy24); } + yymsp[-8].minor.yy360 = yylhsminor.yy360; break; - case 66: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy372 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 101: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy360 = createDropTableClause(pCxt, yymsp[-1].minor.yy345, yymsp[0].minor.yy360); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 67: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy372 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 102: /* specific_tags_opt ::= */ + case 133: /* tags_def_opt ::= */ yytestcase(yyruleno==133); + case 278: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==278); + case 295: /* group_by_clause_opt ::= */ yytestcase(yyruleno==295); + case 305: /* order_by_clause_opt ::= */ yytestcase(yyruleno==305); +{ yymsp[1].minor.yy24 = NULL; } break; - case 68: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy372 = createDataType(TSDB_DATA_TYPE_UINT); } + case 103: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ +{ yymsp[-2].minor.yy24 = yymsp[-1].minor.yy24; } break; - case 69: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy372 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 104: /* full_table_name ::= table_name */ +{ yylhsminor.yy360 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy417, NULL); } + yymsp[0].minor.yy360 = yylhsminor.yy360; break; - case 70: /* type_name ::= JSON */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_JSON); } + case 105: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy360 = createRealTableNode(pCxt, &yymsp[-2].minor.yy417, &yymsp[0].minor.yy417, NULL); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 71: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy372 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 107: /* column_def_list ::= column_def_list NK_COMMA column_def */ + case 148: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==148); + case 170: /* func_name_list ::= func_name_list NK_COMMA col_name */ yytestcase(yyruleno==170); + case 179: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==179); + case 271: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==271); + case 318: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==318); +{ yylhsminor.yy24 = addNodeToList(pCxt, yymsp[-2].minor.yy24, yymsp[0].minor.yy360); } + yymsp[-2].minor.yy24 = yylhsminor.yy24; break; - case 72: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 108: /* column_def ::= column_name type_name */ +{ yylhsminor.yy360 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy417, yymsp[0].minor.yy400, NULL); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 73: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 109: /* column_def ::= column_name type_name COMMENT NK_STRING */ +{ yylhsminor.yy360 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy417, yymsp[-2].minor.yy400, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy360 = yylhsminor.yy360; break; - case 74: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy372 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 110: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 75: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy372 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 111: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 76: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy372 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 112: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 77: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy372 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 113: /* type_name ::= INT */ + case 114: /* type_name ::= INTEGER */ yytestcase(yyruleno==114); +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 79: /* tags_def_opt ::= tags_def */ - case 191: /* select_list ::= select_sublist */ yytestcase(yyruleno==191); -{ yylhsminor.yy40 = yymsp[0].minor.yy40; } - yymsp[0].minor.yy40 = yylhsminor.yy40; + case 115: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 80: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy40 = yymsp[-1].minor.yy40; } + case 116: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 81: /* table_options ::= */ -{ yymsp[1].minor.yy418 = createDefaultTableOptions(pCxt);} + case 117: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 82: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy418 = setTableOption(pCxt, yymsp[-2].minor.yy418, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy418 = yylhsminor.yy418; + case 118: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 83: /* table_options ::= table_options KEEP NK_INTEGER */ -{ yylhsminor.yy418 = setTableOption(pCxt, yymsp[-2].minor.yy418, TABLE_OPTION_KEEP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy418 = yylhsminor.yy418; + case 119: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 84: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy418 = setTableOption(pCxt, yymsp[-2].minor.yy418, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy418 = yylhsminor.yy418; + case 120: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 85: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy418 = setTableSmaOption(pCxt, yymsp[-4].minor.yy418, yymsp[-1].minor.yy40); } - yymsp[-4].minor.yy418 = yylhsminor.yy418; + case 121: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 88: /* col_name ::= column_name */ -{ yylhsminor.yy68 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy5); } - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 122: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 89: /* cmd ::= SHOW DNODES */ + case 123: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UINT); } + break; + case 124: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + break; + case 125: /* type_name ::= JSON */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_JSON); } + break; + case 126: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + break; + case 127: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + break; + case 128: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BLOB); } + break; + case 129: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + break; + case 130: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + break; + case 131: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + break; + case 132: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + break; + case 134: /* tags_def_opt ::= tags_def */ + case 269: /* select_list ::= select_sublist */ yytestcase(yyruleno==269); +{ yylhsminor.yy24 = yymsp[0].minor.yy24; } + yymsp[0].minor.yy24 = yylhsminor.yy24; + break; + case 135: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ +{ yymsp[-3].minor.yy24 = yymsp[-1].minor.yy24; } + break; + case 136: /* table_options ::= */ +{ yymsp[1].minor.yy360 = createDefaultTableOptions(pCxt); } + break; + case 137: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy360 = setTableOption(pCxt, yymsp[-2].minor.yy360, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; + break; + case 138: /* table_options ::= table_options KEEP NK_INTEGER */ +{ yylhsminor.yy360 = setTableOption(pCxt, yymsp[-2].minor.yy360, TABLE_OPTION_KEEP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; + break; + case 139: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy360 = setTableOption(pCxt, yymsp[-2].minor.yy360, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; + break; + case 140: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy360 = setTableSmaOption(pCxt, yymsp[-4].minor.yy360, yymsp[-1].minor.yy24); } + yymsp[-4].minor.yy360 = yylhsminor.yy360; + break; + case 141: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ +{ yylhsminor.yy360 = setTableRollupOption(pCxt, yymsp[-4].minor.yy360, yymsp[-1].minor.yy24); } + yymsp[-4].minor.yy360 = yylhsminor.yy360; + break; + case 142: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy360 = createDefaultAlterTableOptions(pCxt); yylhsminor.yy360 = setTableOption(pCxt, yylhsminor.yy360, yymsp[0].minor.yy285.type, &yymsp[0].minor.yy285.val); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 143: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy360 = setTableOption(pCxt, yymsp[-1].minor.yy360, yymsp[0].minor.yy285.type, &yymsp[0].minor.yy285.val); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; + break; + case 144: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy285.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } + break; + case 145: /* alter_table_option ::= KEEP NK_INTEGER */ +{ yymsp[-1].minor.yy285.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } + break; + case 146: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy285.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy285.val = yymsp[0].minor.yy0; } + break; + case 149: /* col_name ::= column_name */ +{ yylhsminor.yy360 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy417); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 150: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL, NULL); } break; - case 90: /* cmd ::= SHOW USERS */ + case 151: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL, NULL); } break; - case 91: /* cmd ::= SHOW DATABASES */ + case 152: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL, NULL); } break; - case 92: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy68, yymsp[0].minor.yy68); } + case 153: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy360, yymsp[0].minor.yy360); } break; - case 93: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy68, yymsp[0].minor.yy68); } + case 154: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy360, yymsp[0].minor.yy360); } break; - case 94: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy68, NULL); } + case 155: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy360, NULL); } break; - case 95: /* cmd ::= SHOW MNODES */ + case 156: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL, NULL); } break; - case 96: /* cmd ::= SHOW MODULES */ + case 157: /* cmd ::= SHOW MODULES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MODULES_STMT, NULL, NULL); } break; - case 97: /* cmd ::= SHOW QNODES */ + case 158: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL, NULL); } break; - case 98: /* cmd ::= SHOW FUNCTIONS */ + case 159: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } break; - case 99: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy68, yymsp[0].minor.yy68); } + case 160: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy360, yymsp[0].minor.yy360); } break; - case 100: /* cmd ::= SHOW STREAMS */ + case 161: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } break; - case 101: /* db_name_cond_opt ::= */ - case 106: /* from_db_opt ::= */ yytestcase(yyruleno==106); -{ yymsp[1].minor.yy68 = createDefaultDatabaseCondValue(pCxt); } - break; - case 102: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy68 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy5); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; - break; - case 103: /* like_pattern_opt ::= */ - case 198: /* where_clause_opt ::= */ yytestcase(yyruleno==198); - case 202: /* twindow_clause_opt ::= */ yytestcase(yyruleno==202); - case 207: /* sliding_opt ::= */ yytestcase(yyruleno==207); - case 209: /* fill_opt ::= */ yytestcase(yyruleno==209); - case 221: /* having_clause_opt ::= */ yytestcase(yyruleno==221); - case 229: /* slimit_clause_opt ::= */ yytestcase(yyruleno==229); - case 233: /* limit_clause_opt ::= */ yytestcase(yyruleno==233); -{ yymsp[1].minor.yy68 = NULL; } - break; - case 104: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy68 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - break; - case 105: /* table_name_cond ::= table_name */ -{ yylhsminor.yy68 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy5); } - yymsp[0].minor.yy68 = yylhsminor.yy68; - break; - case 107: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy68 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy5); } - break; - case 108: /* cmd ::= query_expression */ -{ pCxt->pRootNode = yymsp[0].minor.yy68; } - break; - case 109: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy68 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy68 = yylhsminor.yy68; - break; - case 110: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy68 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy68 = yylhsminor.yy68; - break; - case 111: /* literal ::= NK_STRING */ -{ yylhsminor.yy68 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy68 = yylhsminor.yy68; - break; - case 112: /* literal ::= NK_BOOL */ -{ yylhsminor.yy68 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy68 = yylhsminor.yy68; - break; - case 113: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; - break; - case 114: /* literal ::= duration_literal */ - case 125: /* expression ::= literal */ yytestcase(yyruleno==125); - case 126: /* expression ::= column_reference */ yytestcase(yyruleno==126); - case 129: /* expression ::= subquery */ yytestcase(yyruleno==129); - case 161: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==161); - case 165: /* boolean_primary ::= predicate */ yytestcase(yyruleno==165); - case 167: /* common_expression ::= expression */ yytestcase(yyruleno==167); - case 168: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==168); - case 170: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==170); - case 172: /* table_reference ::= table_primary */ yytestcase(yyruleno==172); - case 173: /* table_reference ::= joined_table */ yytestcase(yyruleno==173); - case 177: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==177); - case 224: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==224); - case 226: /* query_primary ::= query_specification */ yytestcase(yyruleno==226); -{ yylhsminor.yy68 = yymsp[0].minor.yy68; } - yymsp[0].minor.yy68 = yylhsminor.yy68; - break; - case 115: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy68 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy68 = yylhsminor.yy68; - break; - case 116: /* literal_list ::= literal */ - case 138: /* expression_list ::= expression */ yytestcase(yyruleno==138); -{ yylhsminor.yy40 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy68)); } - yymsp[0].minor.yy40 = yylhsminor.yy40; - break; - case 117: /* literal_list ::= literal_list NK_COMMA literal */ - case 139: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==139); -{ yylhsminor.yy40 = addNodeToList(pCxt, yymsp[-2].minor.yy40, releaseRawExprNode(pCxt, yymsp[0].minor.yy68)); } - yymsp[-2].minor.yy40 = yylhsminor.yy40; - break; - case 127: /* expression ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy5, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy5, yymsp[-1].minor.yy40)); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; - break; - case 128: /* expression ::= function_name NK_LP NK_STAR NK_RP */ -{ yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy5, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy5, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; - break; - case 130: /* expression ::= NK_LP expression NK_RP */ - case 166: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==166); -{ yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy68)); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; - break; - case 131: /* expression ::= NK_PLUS expression */ + case 162: /* db_name_cond_opt ::= */ + case 167: /* from_db_opt ::= */ yytestcase(yyruleno==167); +{ yymsp[1].minor.yy360 = createDefaultDatabaseCondValue(pCxt); } + break; + case 163: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy360 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy417); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; + break; + case 164: /* like_pattern_opt ::= */ + case 175: /* index_options ::= */ yytestcase(yyruleno==175); + case 276: /* where_clause_opt ::= */ yytestcase(yyruleno==276); + case 280: /* twindow_clause_opt ::= */ yytestcase(yyruleno==280); + case 285: /* sliding_opt ::= */ yytestcase(yyruleno==285); + case 287: /* fill_opt ::= */ yytestcase(yyruleno==287); + case 299: /* having_clause_opt ::= */ yytestcase(yyruleno==299); + case 307: /* slimit_clause_opt ::= */ yytestcase(yyruleno==307); + case 311: /* limit_clause_opt ::= */ yytestcase(yyruleno==311); +{ yymsp[1].minor.yy360 = NULL; } + break; + case 165: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy360 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + break; + case 166: /* table_name_cond ::= table_name */ +{ yylhsminor.yy360 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy417); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 168: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy360 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy417); } + break; + case 171: /* func_name ::= function_name */ +{ yylhsminor.yy360 = createFunctionNode(pCxt, &yymsp[0].minor.yy417, NULL); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 172: /* cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &yymsp[-3].minor.yy417, &yymsp[-1].minor.yy417, NULL, yymsp[0].minor.yy360); } + break; + case 173: /* cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &yymsp[-5].minor.yy417, &yymsp[-3].minor.yy417, yymsp[-1].minor.yy24, NULL); } + break; + case 174: /* cmd ::= DROP INDEX index_name ON table_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, &yymsp[-2].minor.yy417, &yymsp[0].minor.yy417); } + break; + case 176: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ +{ yymsp[-8].minor.yy360 = createIndexOption(pCxt, yymsp[-6].minor.yy24, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), NULL, yymsp[0].minor.yy360); } + break; + case 177: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ +{ yymsp[-10].minor.yy360 = createIndexOption(pCxt, yymsp[-8].minor.yy24, releaseRawExprNode(pCxt, yymsp[-4].minor.yy360), releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), yymsp[0].minor.yy360); } + break; + case 180: /* func ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy360 = createFunctionNode(pCxt, &yymsp[-3].minor.yy417, yymsp[-1].minor.yy24); } + yymsp[-3].minor.yy360 = yylhsminor.yy360; + break; + case 181: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy345, &yymsp[-2].minor.yy417, yymsp[0].minor.yy360, NULL); } + break; + case 182: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy345, &yymsp[-2].minor.yy417, NULL, &yymsp[0].minor.yy417); } + break; + case 183: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy345, &yymsp[0].minor.yy417); } + break; + case 185: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy360 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 186: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy360 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 187: /* literal ::= NK_STRING */ +{ yylhsminor.yy360 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 188: /* literal ::= NK_BOOL */ +{ yylhsminor.yy360 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 189: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; + break; + case 190: /* literal ::= duration_literal */ + case 203: /* expression ::= literal */ yytestcase(yyruleno==203); + case 204: /* expression ::= column_reference */ yytestcase(yyruleno==204); + case 207: /* expression ::= subquery */ yytestcase(yyruleno==207); + case 239: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==239); + case 243: /* boolean_primary ::= predicate */ yytestcase(yyruleno==243); + case 245: /* common_expression ::= expression */ yytestcase(yyruleno==245); + case 246: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==246); + case 248: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==248); + case 250: /* table_reference ::= table_primary */ yytestcase(yyruleno==250); + case 251: /* table_reference ::= joined_table */ yytestcase(yyruleno==251); + case 255: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==255); + case 302: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==302); + case 304: /* query_primary ::= query_specification */ yytestcase(yyruleno==304); +{ yylhsminor.yy360 = yymsp[0].minor.yy360; } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 191: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy360 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy360 = yylhsminor.yy360; + break; + case 192: /* literal_list ::= literal */ + case 216: /* expression_list ::= expression */ yytestcase(yyruleno==216); +{ yylhsminor.yy24 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy360)); } + yymsp[0].minor.yy24 = yylhsminor.yy24; + break; + case 193: /* literal_list ::= literal_list NK_COMMA literal */ + case 217: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==217); +{ yylhsminor.yy24 = addNodeToList(pCxt, yymsp[-2].minor.yy24, releaseRawExprNode(pCxt, yymsp[0].minor.yy360)); } + yymsp[-2].minor.yy24 = yylhsminor.yy24; + break; + case 205: /* expression ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy417, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy417, yymsp[-1].minor.yy24)); } + yymsp[-3].minor.yy360 = yylhsminor.yy360; + break; + case 206: /* expression ::= function_name NK_LP NK_STAR NK_RP */ +{ yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy417, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy417, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } + yymsp[-3].minor.yy360 = yylhsminor.yy360; + break; + case 208: /* expression ::= NK_LP expression NK_RP */ + case 244: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==244); +{ yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy360)); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; + break; + case 209: /* expression ::= NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy68)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy360)); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 132: /* expression ::= NK_MINUS expression */ + case 210: /* expression ::= NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy68), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy360), NULL)); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 133: /* expression ::= expression NK_PLUS expression */ + case 211: /* expression ::= expression NK_PLUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 134: /* expression ::= expression NK_MINUS expression */ + case 212: /* expression ::= expression NK_MINUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 135: /* expression ::= expression NK_STAR expression */ + case 213: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 136: /* expression ::= expression NK_SLASH expression */ + case 214: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 137: /* expression ::= expression NK_REM expression */ + case 215: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 140: /* column_reference ::= column_name */ -{ yylhsminor.yy68 = createRawExprNode(pCxt, &yymsp[0].minor.yy5, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy5)); } - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 218: /* column_reference ::= column_name */ +{ yylhsminor.yy360 = createRawExprNode(pCxt, &yymsp[0].minor.yy417, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy417)); } + yymsp[0].minor.yy360 = yylhsminor.yy360; break; - case 141: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy5, &yymsp[0].minor.yy5, createColumnNode(pCxt, &yymsp[-2].minor.yy5, &yymsp[0].minor.yy5)); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 219: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy417, &yymsp[0].minor.yy417, createColumnNode(pCxt, &yymsp[-2].minor.yy417, &yymsp[0].minor.yy417)); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 142: /* predicate ::= expression compare_op expression */ - case 147: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==147); + case 220: /* predicate ::= expression compare_op expression */ + case 225: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==225); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy416, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy252, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 143: /* predicate ::= expression BETWEEN expression AND expression */ + case 221: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy68), releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy360), releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-4].minor.yy68 = yylhsminor.yy68; + yymsp[-4].minor.yy360 = yylhsminor.yy360; break; - case 144: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 222: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[-5].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[-5].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-5].minor.yy68 = yylhsminor.yy68; + yymsp[-5].minor.yy360 = yylhsminor.yy360; break; - case 145: /* predicate ::= expression IS NULL */ + case 223: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), NULL)); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 146: /* predicate ::= expression IS NOT NULL */ + case 224: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy68), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy360), NULL)); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; + yymsp[-3].minor.yy360 = yylhsminor.yy360; break; - case 148: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy416 = OP_TYPE_LOWER_THAN; } + case 226: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy252 = OP_TYPE_LOWER_THAN; } break; - case 149: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy416 = OP_TYPE_GREATER_THAN; } + case 227: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy252 = OP_TYPE_GREATER_THAN; } break; - case 150: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy416 = OP_TYPE_LOWER_EQUAL; } + case 228: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy252 = OP_TYPE_LOWER_EQUAL; } break; - case 151: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy416 = OP_TYPE_GREATER_EQUAL; } + case 229: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy252 = OP_TYPE_GREATER_EQUAL; } break; - case 152: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy416 = OP_TYPE_NOT_EQUAL; } + case 230: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy252 = OP_TYPE_NOT_EQUAL; } break; - case 153: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy416 = OP_TYPE_EQUAL; } + case 231: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy252 = OP_TYPE_EQUAL; } break; - case 154: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy416 = OP_TYPE_LIKE; } + case 232: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy252 = OP_TYPE_LIKE; } break; - case 155: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy416 = OP_TYPE_NOT_LIKE; } + case 233: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy252 = OP_TYPE_NOT_LIKE; } break; - case 156: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy416 = OP_TYPE_MATCH; } + case 234: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy252 = OP_TYPE_MATCH; } break; - case 157: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy416 = OP_TYPE_NMATCH; } + case 235: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy252 = OP_TYPE_NMATCH; } break; - case 158: /* in_op ::= IN */ -{ yymsp[0].minor.yy416 = OP_TYPE_IN; } + case 236: /* in_op ::= IN */ +{ yymsp[0].minor.yy252 = OP_TYPE_IN; } break; - case 159: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy416 = OP_TYPE_NOT_IN; } + case 237: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy252 = OP_TYPE_NOT_IN; } break; - case 160: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy40)); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 238: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy24)); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 162: /* boolean_value_expression ::= NOT boolean_primary */ + case 240: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy68), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy360), NULL)); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 163: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 241: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 164: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 242: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy68); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy360); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 169: /* from_clause ::= FROM table_reference_list */ - case 199: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==199); - case 222: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==222); -{ yymsp[-1].minor.yy68 = yymsp[0].minor.yy68; } + case 247: /* from_clause ::= FROM table_reference_list */ + case 277: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==277); + case 300: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==300); +{ yymsp[-1].minor.yy360 = yymsp[0].minor.yy360; } break; - case 171: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy68 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy68, yymsp[0].minor.yy68, NULL); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 249: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy360 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy360, yymsp[0].minor.yy360, NULL); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 174: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy68 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy5, &yymsp[0].minor.yy5); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; + case 252: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy360 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy417, &yymsp[0].minor.yy417); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 175: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy68 = createRealTableNode(pCxt, &yymsp[-3].minor.yy5, &yymsp[-1].minor.yy5, &yymsp[0].minor.yy5); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; + case 253: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy360 = createRealTableNode(pCxt, &yymsp[-3].minor.yy417, &yymsp[-1].minor.yy417, &yymsp[0].minor.yy417); } + yymsp[-3].minor.yy360 = yylhsminor.yy360; break; - case 176: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy68 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy68), &yymsp[0].minor.yy5); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; + case 254: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy360 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy360), &yymsp[0].minor.yy417); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 178: /* alias_opt ::= */ -{ yymsp[1].minor.yy5 = nil_token; } + case 256: /* alias_opt ::= */ +{ yymsp[1].minor.yy417 = nil_token; } break; - case 179: /* alias_opt ::= table_alias */ -{ yylhsminor.yy5 = yymsp[0].minor.yy5; } - yymsp[0].minor.yy5 = yylhsminor.yy5; + case 257: /* alias_opt ::= table_alias */ +{ yylhsminor.yy417 = yymsp[0].minor.yy417; } + yymsp[0].minor.yy417 = yylhsminor.yy417; break; - case 180: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy5 = yymsp[0].minor.yy5; } + case 258: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy417 = yymsp[0].minor.yy417; } break; - case 181: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 182: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==182); -{ yymsp[-2].minor.yy68 = yymsp[-1].minor.yy68; } + case 259: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 260: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==260); +{ yymsp[-2].minor.yy360 = yymsp[-1].minor.yy360; } break; - case 183: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy68 = createJoinTableNode(pCxt, yymsp[-4].minor.yy92, yymsp[-5].minor.yy68, yymsp[-2].minor.yy68, yymsp[0].minor.yy68); } - yymsp[-5].minor.yy68 = yylhsminor.yy68; + case 261: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy360 = createJoinTableNode(pCxt, yymsp[-4].minor.yy84, yymsp[-5].minor.yy360, yymsp[-2].minor.yy360, yymsp[0].minor.yy360); } + yymsp[-5].minor.yy360 = yylhsminor.yy360; break; - case 184: /* join_type ::= */ -{ yymsp[1].minor.yy92 = JOIN_TYPE_INNER; } + case 262: /* join_type ::= */ +{ yymsp[1].minor.yy84 = JOIN_TYPE_INNER; } break; - case 185: /* join_type ::= INNER */ -{ yymsp[0].minor.yy92 = JOIN_TYPE_INNER; } + case 263: /* join_type ::= INNER */ +{ yymsp[0].minor.yy84 = JOIN_TYPE_INNER; } break; - case 186: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 264: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-8].minor.yy68 = createSelectStmt(pCxt, yymsp[-7].minor.yy25, yymsp[-6].minor.yy40, yymsp[-5].minor.yy68); - yymsp[-8].minor.yy68 = addWhereClause(pCxt, yymsp[-8].minor.yy68, yymsp[-4].minor.yy68); - yymsp[-8].minor.yy68 = addPartitionByClause(pCxt, yymsp[-8].minor.yy68, yymsp[-3].minor.yy40); - yymsp[-8].minor.yy68 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy68, yymsp[-2].minor.yy68); - yymsp[-8].minor.yy68 = addGroupByClause(pCxt, yymsp[-8].minor.yy68, yymsp[-1].minor.yy40); - yymsp[-8].minor.yy68 = addHavingClause(pCxt, yymsp[-8].minor.yy68, yymsp[0].minor.yy68); + yymsp[-8].minor.yy360 = createSelectStmt(pCxt, yymsp[-7].minor.yy345, yymsp[-6].minor.yy24, yymsp[-5].minor.yy360); + yymsp[-8].minor.yy360 = addWhereClause(pCxt, yymsp[-8].minor.yy360, yymsp[-4].minor.yy360); + yymsp[-8].minor.yy360 = addPartitionByClause(pCxt, yymsp[-8].minor.yy360, yymsp[-3].minor.yy24); + yymsp[-8].minor.yy360 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy360, yymsp[-2].minor.yy360); + yymsp[-8].minor.yy360 = addGroupByClause(pCxt, yymsp[-8].minor.yy360, yymsp[-1].minor.yy24); + yymsp[-8].minor.yy360 = addHavingClause(pCxt, yymsp[-8].minor.yy360, yymsp[0].minor.yy360); } break; - case 188: /* set_quantifier_opt ::= DISTINCT */ -{ yymsp[0].minor.yy25 = true; } + case 266: /* set_quantifier_opt ::= DISTINCT */ +{ yymsp[0].minor.yy345 = true; } break; - case 189: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy25 = false; } + case 267: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy345 = false; } break; - case 190: /* select_list ::= NK_STAR */ -{ yymsp[0].minor.yy40 = NULL; } + case 268: /* select_list ::= NK_STAR */ +{ yymsp[0].minor.yy24 = NULL; } break; - case 194: /* select_item ::= common_expression */ + case 272: /* select_item ::= common_expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy68); - yylhsminor.yy68 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy68), &t); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy360); + yylhsminor.yy360 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy360), &t); } - yymsp[0].minor.yy68 = yylhsminor.yy68; + yymsp[0].minor.yy360 = yylhsminor.yy360; break; - case 195: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy68 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy68), &yymsp[0].minor.yy5); } - yymsp[-1].minor.yy68 = yylhsminor.yy68; + case 273: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy360 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy360), &yymsp[0].minor.yy417); } + yymsp[-1].minor.yy360 = yylhsminor.yy360; break; - case 196: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy68 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), &yymsp[0].minor.yy5); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 274: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy360 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), &yymsp[0].minor.yy417); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 197: /* select_item ::= table_name NK_DOT NK_STAR */ -{ yylhsminor.yy68 = createColumnNode(pCxt, &yymsp[-2].minor.yy5, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 275: /* select_item ::= table_name NK_DOT NK_STAR */ +{ yylhsminor.yy360 = createColumnNode(pCxt, &yymsp[-2].minor.yy417, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 201: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 218: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==218); - case 228: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==228); -{ yymsp[-2].minor.yy40 = yymsp[0].minor.yy40; } + case 279: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 296: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==296); + case 306: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==306); +{ yymsp[-2].minor.yy24 = yymsp[0].minor.yy24; } break; - case 203: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy68 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy68), &yymsp[-1].minor.yy0); } + case 281: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy360 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy360), &yymsp[-1].minor.yy0); } break; - case 204: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ -{ yymsp[-3].minor.yy68 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy68)); } + case 282: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ yymsp[-3].minor.yy360 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy360)); } break; - case 205: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy68 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy68, NULL, yymsp[-1].minor.yy68, yymsp[0].minor.yy68); } + case 283: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy360 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy360), NULL, yymsp[-1].minor.yy360, yymsp[0].minor.yy360); } break; - case 206: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy68 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy68, yymsp[-3].minor.yy68, yymsp[-1].minor.yy68, yymsp[0].minor.yy68); } + case 284: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy360 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy360), releaseRawExprNode(pCxt, yymsp[-3].minor.yy360), yymsp[-1].minor.yy360, yymsp[0].minor.yy360); } break; - case 208: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ yymsp[-3].minor.yy68 = yymsp[-1].minor.yy68; } + case 286: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ yymsp[-3].minor.yy360 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy360); } break; - case 210: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy68 = createFillNode(pCxt, yymsp[-1].minor.yy94, NULL); } + case 288: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy360 = createFillNode(pCxt, yymsp[-1].minor.yy358, NULL); } break; - case 211: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy68 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy40)); } + case 289: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy360 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy24)); } break; - case 212: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy94 = FILL_MODE_NONE; } + case 290: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy358 = FILL_MODE_NONE; } break; - case 213: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy94 = FILL_MODE_PREV; } + case 291: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy358 = FILL_MODE_PREV; } break; - case 214: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy94 = FILL_MODE_NULL; } + case 292: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy358 = FILL_MODE_NULL; } break; - case 215: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy94 = FILL_MODE_LINEAR; } + case 293: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy358 = FILL_MODE_LINEAR; } break; - case 216: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy94 = FILL_MODE_NEXT; } + case 294: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy358 = FILL_MODE_NEXT; } break; - case 219: /* group_by_list ::= expression */ -{ yylhsminor.yy40 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); } - yymsp[0].minor.yy40 = yylhsminor.yy40; + case 297: /* group_by_list ::= expression */ +{ yylhsminor.yy24 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } + yymsp[0].minor.yy24 = yylhsminor.yy24; break; - case 220: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy40 = addNodeToList(pCxt, yymsp[-2].minor.yy40, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy68))); } - yymsp[-2].minor.yy40 = yylhsminor.yy40; + case 298: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy24 = addNodeToList(pCxt, yymsp[-2].minor.yy24, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy360))); } + yymsp[-2].minor.yy24 = yylhsminor.yy24; break; - case 223: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 301: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy68 = addOrderByClause(pCxt, yymsp[-3].minor.yy68, yymsp[-2].minor.yy40); - yylhsminor.yy68 = addSlimitClause(pCxt, yylhsminor.yy68, yymsp[-1].minor.yy68); - yylhsminor.yy68 = addLimitClause(pCxt, yylhsminor.yy68, yymsp[0].minor.yy68); + yylhsminor.yy360 = addOrderByClause(pCxt, yymsp[-3].minor.yy360, yymsp[-2].minor.yy24); + yylhsminor.yy360 = addSlimitClause(pCxt, yylhsminor.yy360, yymsp[-1].minor.yy360); + yylhsminor.yy360 = addLimitClause(pCxt, yylhsminor.yy360, yymsp[0].minor.yy360); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; + yymsp[-3].minor.yy360 = yylhsminor.yy360; break; - case 225: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy68 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy68, yymsp[0].minor.yy68); } - yymsp[-3].minor.yy68 = yylhsminor.yy68; + case 303: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy360 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy360, yymsp[0].minor.yy360); } + yymsp[-3].minor.yy360 = yylhsminor.yy360; break; - case 230: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 234: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==234); -{ yymsp[-1].minor.yy68 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 308: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 312: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==312); +{ yymsp[-1].minor.yy360 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 231: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 235: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==235); -{ yymsp[-3].minor.yy68 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 309: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 313: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==313); +{ yymsp[-3].minor.yy360 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 232: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 236: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==236); -{ yymsp[-3].minor.yy68 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 310: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 314: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==314); +{ yymsp[-3].minor.yy360 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 237: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy68 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy68); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 315: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy360 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy360); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 238: /* search_condition ::= common_expression */ -{ yylhsminor.yy68 = releaseRawExprNode(pCxt, yymsp[0].minor.yy68); } - yymsp[0].minor.yy68 = yylhsminor.yy68; + case 316: /* search_condition ::= common_expression */ +{ yylhsminor.yy360 = releaseRawExprNode(pCxt, yymsp[0].minor.yy360); } + yymsp[0].minor.yy360 = yylhsminor.yy360; break; - case 241: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy68 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy68), yymsp[-1].minor.yy54, yymsp[0].minor.yy53); } - yymsp[-2].minor.yy68 = yylhsminor.yy68; + case 319: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy360 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy360), yymsp[-1].minor.yy130, yymsp[0].minor.yy73); } + yymsp[-2].minor.yy360 = yylhsminor.yy360; break; - case 242: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy54 = ORDER_ASC; } + case 320: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy130 = ORDER_ASC; } break; - case 243: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy54 = ORDER_ASC; } + case 321: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy130 = ORDER_ASC; } break; - case 244: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy54 = ORDER_DESC; } + case 322: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy130 = ORDER_DESC; } break; - case 245: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy53 = NULL_ORDER_DEFAULT; } + case 323: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy73 = NULL_ORDER_DEFAULT; } break; - case 246: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy53 = NULL_ORDER_FIRST; } + case 324: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy73 = NULL_ORDER_FIRST; } break; - case 247: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy53 = NULL_ORDER_LAST; } + case 325: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy73 = NULL_ORDER_LAST; } break; default: break; @@ -2788,13 +3304,15 @@ static void yy_syntax_error( ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ - - if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); - } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + + if (pCxt->valid) { + if(TOKEN.z) { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); + } else { + generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + } + pCxt->valid = false; } - pCxt->valid = false; /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ ParseCTX_STORE diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 65d3f51dde3e1aaf7cba570f60b9f3f10bf5641c..1f333e49e7726f90567d437731c27b878e449046 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -13,12 +13,11 @@ * along with this program. If not, see . */ -#include "mockCatalogService.h" - #include #include #include #include "tdatablock.h" +#include "mockCatalogService.h" #include "tname.h" #include "ttypes.h" diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index b4748a473678d77d75ad5c7d01138905b922b539..8079e4f3bdd0a95767b05ea6e6b03e379956b825 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -309,6 +309,13 @@ TEST_F(ParserTest, showUsers) { ASSERT_TRUE(run()); } +TEST_F(ParserTest, alterAccount) { + setDatabase("root", "test"); + + bind("alter account ac_wxy pass '123456'"); + ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); +} + TEST_F(ParserTest, createDnode) { setDatabase("root", "test"); @@ -326,6 +333,16 @@ TEST_F(ParserTest, showDnodes) { ASSERT_TRUE(run()); } +TEST_F(ParserTest, alterDnode) { + setDatabase("root", "test"); + + bind("alter dnode 1 'resetLog'"); + ASSERT_TRUE(run()); + + bind("alter dnode 1 'debugFlag' '134'"); + ASSERT_TRUE(run()); +} + TEST_F(ParserTest, createDatabase) { setDatabase("root", "test"); @@ -354,7 +371,24 @@ TEST_F(ParserTest, createDatabase) { ASSERT_TRUE(run()); } -TEST_F(ParserTest, showDatabases) { +TEST_F(ParserTest, alterDatabase) { + setDatabase("root", "test"); + + bind("alter database wxy_db BLOCKS 200"); + ASSERT_TRUE(run()); + + bind("alter database wxy_db " + "BLOCKS 200 " + "CACHELAST 1 " + "FSYNC 200 " + "KEEP 200 " + "QUORUM 2 " + "WAL 1 " + ); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, showDatabase) { setDatabase("root", "test"); bind("show databases"); @@ -499,3 +533,57 @@ TEST_F(ParserTest, showStreams) { bind("show streams"); ASSERT_TRUE(run()); } + +TEST_F(ParserTest, createSmaIndex) { + setDatabase("root", "test"); + + bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, dropIndex) { + setDatabase("root", "test"); + + bind("drop index index1 on t1"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, createQnode) { + setDatabase("root", "test"); + + bind("create qnode on dnode 1"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, dropQnode) { + setDatabase("root", "test"); + + bind("drop qnode on dnode 1"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, createTopic) { + setDatabase("root", "test"); + + bind("create topic tp1 as select * from t1"); + ASSERT_TRUE(run()); + + bind("create topic if not exists tp1 as select * from t1"); + ASSERT_TRUE(run()); + + bind("create topic tp1 as test"); + ASSERT_TRUE(run()); + + bind("create topic if not exists tp1 as test"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, dropTopic) { + setDatabase("root", "test"); + + bind("drop topic tp1"); + ASSERT_TRUE(run()); + + bind("drop topic if exists tp1"); + ASSERT_TRUE(run()); +} diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ff64150f68e18e6547ee46c22eed21e67fc7120c..881db88503f26c7902d665d7d12d0d435a3cc085 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -18,13 +18,13 @@ #include "functionMgt.h" typedef struct SLogicPlanContext { - int32_t errCode; - int32_t planNodeId; - int32_t acctId; + SPlanContext* pPlanCxt; } SLogicPlanContext; -static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt); -static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable); +typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**); + +static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode); +static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode); typedef struct SRewriteExprCxt { int32_t errCode; @@ -66,7 +66,6 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { } typedef struct SNameExprCxt { - int32_t planNodeId; int32_t rewriteId; } SNameExprCxt; @@ -76,7 +75,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { SNameExprCxt* pCxt = (SNameExprCxt*)pContext; - sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId++); + sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d", pCxt->rewriteId++); return DEAL_RES_IGNORE_CHILD; } default: @@ -86,44 +85,49 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - SNameExprCxt nameCxt = { .planNodeId = planNodeId, .rewriteId = rewriteId }; +static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { + static int32_t rewriteId = 1; + SNameExprCxt nameCxt = { .rewriteId = rewriteId }; nodesWalkList(pExprs, doNameExpr, &nameCxt); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; } -static SLogicNode* pushLogicNode(SLogicPlanContext* pCxt, SLogicNode* pRoot, SLogicNode* pNode) { - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - goto error; +static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) { + if (NULL == pNewRoot->pChildren) { + pNewRoot->pChildren = nodesMakeList(); + if (NULL == pNewRoot->pChildren) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - - if (NULL == pRoot) { - return pNode; + if (TSDB_CODE_SUCCESS != nodesListAppend(pNewRoot->pChildren, (SNode*)*pOldRoot)) { + return TSDB_CODE_OUT_OF_MEMORY; } - if (NULL == pNode) { - return pRoot; - } + (*pOldRoot)->pParent = pNewRoot; + *pOldRoot = pNewRoot; - if (NULL == pNode->pChildren) { - pNode->pChildren = nodesMakeList(); - if (NULL == pNode->pChildren) { - goto error; - } + return TSDB_CODE_SUCCESS; +} + +static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, FCreateLogicNode func, SLogicNode** pRoot) { + SLogicNode* pNode = NULL; + int32_t code = func(pCxt, pSelect, &pNode); + if (TSDB_CODE_SUCCESS == code && NULL != pNode) { + code = pushLogicNode(pCxt, pRoot, pNode); } - if (TSDB_CODE_SUCCESS != nodesListAppend(pNode->pChildren, (SNode*)pRoot)) { - goto error; + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNode); } - pRoot->pParent = pNode; - return pNode; -error: - nodesDestroyNode((SNode*)pNode); - return pRoot; + return code; } -static EScanType getScanType(SNodeList* pScanCols, STableMeta* pMeta) { +static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanCols, STableMeta* pMeta) { + if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { + return SCAN_TYPE_STREAM; + } + if (NULL == pScanCols) { // select count(*) from t return SCAN_TYPE_TABLE; @@ -143,94 +147,157 @@ static EScanType getScanType(SNodeList* pScanCols, STableMeta* pMeta) { return SCAN_TYPE_TAG; } -static SLogicNode* createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable) { +static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) { SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); - CHECK_ALLOC(pScan, NULL); - pScan->node.id = pCxt->planNodeId++; + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*); + pScan->scanFlag = MAIN_SCAN; + pScan->scanRange = TSWINDOW_INITIALIZER; + pScan->tableName.type = TSDB_TABLE_NAME_T; + pScan->tableName.acctId = pCxt->pPlanCxt->acctId; + strcpy(pScan->tableName.dbname, pRealTable->table.dbName); + strcpy(pScan->tableName.tname, pRealTable->table.tableName); // set columns to scan SNodeList* pCols = NULL; - CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan); - if (NULL != pCols) { + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols); + if (TSDB_CODE_SUCCESS == code && NULL != pCols) { pScan->pScanCols = nodesCloneList(pCols); - CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan); + if (NULL == pScan) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } + pScan->scanType = getScanType(pCxt, pCols, pScan->pMeta); + // set output - if (NULL != pCols) { + if (TSDB_CODE_SUCCESS == code && NULL != pCols) { pScan->node.pTargets = nodesCloneList(pCols); - CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan); + if (NULL == pScan) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } - pScan->scanType = getScanType(pCols, pScan->pMeta); - pScan->scanFlag = MAIN_SCAN; - pScan->scanRange = TSWINDOW_INITIALIZER; - pScan->tableName.type = TSDB_TABLE_NAME_T; - pScan->tableName.acctId = pCxt->acctId; - strcpy(pScan->tableName.dbname, pRealTable->table.dbName); - strcpy(pScan->tableName.tname, pRealTable->table.tableName); + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pScan; + } else { + nodesDestroyNode(pScan); + } - return (SLogicNode*)pScan; + return code; } -static SLogicNode* createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable) { - SLogicNode* pRoot = createQueryLogicNode(pCxt, pTable->pSubquery); - CHECK_ALLOC(pRoot, NULL); - SNode* pNode; - FOREACH(pNode, pRoot->pTargets) { - strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias); +static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, SLogicNode** pLogicNode) { + int32_t code = createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode); + if (TSDB_CODE_SUCCESS == code) { + SNode* pNode; + FOREACH(pNode, (*pLogicNode)->pTargets) { + strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias); + } } - return pRoot; + return code; } -static SLogicNode* createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable) { +static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable, SLogicNode** pLogicNode) { SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_JOIN); - CHECK_ALLOC(pJoin, NULL); - pJoin->node.id = pCxt->planNodeId++; + if (NULL == pJoin) { + return TSDB_CODE_OUT_OF_MEMORY; + } pJoin->joinType = pJoinTable->joinType; + int32_t code = TSDB_CODE_SUCCESS; + // set left and right node pJoin->node.pChildren = nodesMakeList(); - CHECK_ALLOC(pJoin->node.pChildren, (SLogicNode*)pJoin); - SLogicNode* pLeft = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft); - CHECK_ALLOC(pLeft, (SLogicNode*)pJoin); - CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pLeft), (SLogicNode*)pJoin); - SLogicNode* pRight = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight); - CHECK_ALLOC(pRight, (SLogicNode*)pJoin); - CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin); + if (NULL == pJoin->node.pChildren) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + SLogicNode* pLeft = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft, &pLeft); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pLeft); + } + } + + SLogicNode* pRight = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight, &pRight); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pRight); + } + } // set on conditions - if (NULL != pJoinTable->pOnCond) { + if (TSDB_CODE_SUCCESS == code && NULL != pJoinTable->pOnCond) { pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); - CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); + if (NULL == pJoin->pOnConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } // set the output - pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); - CHECK_ALLOC(pJoin->node.pTargets, (SLogicNode*)pJoin); - SNodeList* pTargets = nodesCloneList(pRight->pTargets); - CHECK_ALLOC(pTargets, (SLogicNode*)pJoin); - nodesListAppendList(pJoin->node.pTargets, pTargets); + if (TSDB_CODE_SUCCESS == code) { + pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); + if (NULL == pJoin->pOnConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppendList(pJoin->node.pTargets, nodesCloneList(pRight->pTargets)); + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pJoin; + } else { + nodesDestroyNode(pJoin); + } - return (SLogicNode*)pJoin; + return code; } -static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable) { +static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) { switch (nodeType(pTable)) { case QUERY_NODE_REAL_TABLE: - return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable); + return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable, pLogicNode); case QUERY_NODE_TEMP_TABLE: - return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable); + return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable, pLogicNode); case QUERY_NODE_JOIN_TABLE: - return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable); + return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable, pLogicNode); default: break; } - return NULL; + return TSDB_CODE_FAILED; +} + +static int32_t createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) { + SLogicNode* pNode = NULL; + int32_t code = doCreateLogicNodeByTable(pCxt, pSelect, pTable, &pNode); + if (TSDB_CODE_SUCCESS == code) { + pNode->pConditions = nodesCloneNode(pSelect->pWhere); + if (NULL != pSelect->pWhere && NULL == pNode->pConditions) { + nodesDestroyNode(pNode); + return TSDB_CODE_OUT_OF_MEMORY; + } + *pLogicNode = pNode; + } + return code; +} + +static SColumnNode* createColumnByExpr(SExprNode* pExpr) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + pCol->node.resType = pExpr->resType; + strcpy(pCol->colName, pExpr->aliasName); + return pCol; } typedef struct SCreateColumnCxt { @@ -265,192 +332,241 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static SNodeList* createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs) { - SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() }; - CHECK_ALLOC(cxt.pList, NULL); +static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) { + SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList) }; + if (NULL == cxt.pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } nodesWalkList(pExprs, doCreateColumn, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pList); - return NULL; + return cxt.errCode; } - return cxt.pList; + if (NULL == *pList) { + *pList = cxt.pList; + } + return cxt.errCode; } -static SLogicNode* createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SNodeList* pAggFuncs = NULL; - CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs), NULL); + int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs); + if (TSDB_CODE_SUCCESS != code) { + return code; + } if (NULL == pAggFuncs && NULL == pSelect->pGroupByList) { - return NULL; + return TSDB_CODE_SUCCESS; } SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); - CHECK_ALLOC(pAgg, NULL); - pAgg->node.id = pCxt->planNodeId++; + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } // set grouyp keys, agg funcs and having conditions if (NULL != pSelect->pGroupByList) { pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); - CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); + if (NULL == pAgg->pGroupKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } - if (NULL != pAggFuncs) { + + if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) { pAgg->pAggFuncs = nodesCloneList(pAggFuncs); - CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); + if (NULL == pAgg->pAggFuncs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } // rewrite the expression in subsequent clauses - CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); - CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); + } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + } - if (NULL != pSelect->pHaving) { + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); - CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); + if (NULL == pAgg->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } // set the output - pAgg->node.pTargets = nodesMakeList(); - CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); - if (NULL != pAgg->pGroupKeys) { - SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys); - CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); - nodesListAppendList(pAgg->node.pTargets, pTargets); - } - if (NULL != pAgg->pAggFuncs) { - SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); - CHECK_ALLOC(pTargets, (SLogicNode*)pAgg); - nodesListAppendList(pAgg->node.pTargets, pTargets); - } - - return (SLogicNode*)pAgg; + if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) { + code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets); + } + if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { + code = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs, &pAgg->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } + + return code; } -static SLogicNode* createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect) { +static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); - CHECK_ALLOC(pWindow, NULL); - pWindow->node.id = pCxt->planNodeId++; + if (NULL == pWindow) { + return TSDB_CODE_OUT_OF_MEMORY; + } pWindow->winType = WINDOW_TYPE_INTERVAL; pWindow->interval = ((SValueNode*)pInterval->pInterval)->datum.i; + pWindow->intervalUnit = ((SValueNode*)pInterval->pInterval)->unit; pWindow->offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0); - pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : 0); + pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); + pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); + + int32_t code = TSDB_CODE_SUCCESS; + if (NULL != pInterval->pFill) { pWindow->pFill = nodesCloneNode(pInterval->pFill); - CHECK_ALLOC(pWindow->pFill, (SLogicNode*)pWindow); + if (NULL == pWindow->pFill) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs); } - SNodeList* pFuncs = NULL; - CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pFuncs), NULL); - if (NULL != pFuncs) { - pWindow->pFuncs = nodesCloneList(pFuncs); - CHECK_ALLOC(pWindow->pFuncs, (SLogicNode*)pWindow); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); } - CHECK_CODE(rewriteExpr(pWindow->node.id, 1, pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW), (SLogicNode*)pWindow); + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets); + } - pWindow->node.pTargets = createColumnByRewriteExps(pCxt, pWindow->pFuncs); - CHECK_ALLOC(pWindow->node.pTargets, (SLogicNode*)pWindow); + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pWindow; + } else { + nodesDestroyNode(pWindow); + } - return (SLogicNode*)pWindow; + return code; } -static SLogicNode* createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pWindow) { - return NULL; + return TSDB_CODE_SUCCESS; } switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_INTERVAL_WINDOW: - return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect); + return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); default: break; } - return NULL; + return TSDB_CODE_FAILED; } -static SNodeList* createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs) { +static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pCols) { SNodeList* pList = nodesMakeList(); - CHECK_ALLOC(pList, NULL); + if (NULL == pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SNode* pNode; - FOREACH(pNode, pExprs) { - SExprNode* pExpr = (SExprNode*)pNode; - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - goto error; - } - pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); - if (TSDB_CODE_SUCCESS != nodesListAppend(pList, (SNode*)pCol)) { - goto error; + FOREACH(pNode, pExprs) { + if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr((SExprNode*)pNode))) { + nodesDestroyList(pList); + return TSDB_CODE_OUT_OF_MEMORY; } } - return pList; -error: - nodesDestroyList(pList); - return NULL; + + *pCols = pList; + return TSDB_CODE_SUCCESS; } -static SLogicNode* createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); - CHECK_ALLOC(pProject, NULL); - pProject->node.id = pCxt->planNodeId++; + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; pProject->pProjections = nodesCloneList(pSelect->pProjectionList); + if (NULL == pProject->pProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByProjections(pCxt,pSelect->pProjectionList, &pProject->node.pTargets); + } - pProject->node.pTargets = createColumnByProjections(pCxt,pSelect->pProjectionList); - CHECK_ALLOC(pProject->node.pTargets, (SLogicNode*)pProject); + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pProject; + } else { + nodesDestroyNode(pProject); + } - return (SLogicNode*)pProject; + return code; } -static SLogicNode* createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { - SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable); - if (TSDB_CODE_SUCCESS == pCxt->errCode && NULL != pSelect->pWhere) { - pRoot->pConditions = nodesCloneNode(pSelect->pWhere); - CHECK_ALLOC(pRoot->pConditions, pRoot); +static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + SLogicNode* pRoot = NULL; + int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot); + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); } - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pRoot = pushLogicNode(pCxt, pRoot, createWindowLogicNode(pCxt, pSelect)); + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); } - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect)); + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot); } - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pRoot = pushLogicNode(pCxt, pRoot, createProjectLogicNode(pCxt, pSelect)); + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = pRoot; + } else { + nodesDestroyNode(pRoot); } - return pRoot; + + return code; } static int32_t getMsgType(ENodeType sqlType) { return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT; } -static SLogicNode* createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt) { - SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF); - CHECK_ALLOC(pModif, NULL); +static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { + SVnodeModifLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF); + if (NULL == pModif) { + return TSDB_CODE_OUT_OF_MEMORY; + } pModif->pDataBlocks = pStmt->pDataBlocks; pModif->msgType = getMsgType(pStmt->sqlNodeType); - return (SLogicNode*)pModif; + *pLogicNode = (SLogicNode*)pModif; + return TSDB_CODE_SUCCESS; } -static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt) { +static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode) { switch (nodeType(pStmt)) { case QUERY_NODE_SELECT_STMT: - return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt); + return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode); case QUERY_NODE_VNODE_MODIF_STMT: - return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt); + return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode); default: break; } + return TSDB_CODE_FAILED; } int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { - SLogicPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 1, .acctId = pCxt->acctId }; - SLogicNode* pRoot = createQueryLogicNode(&cxt, pCxt->pAstRoot); - if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesDestroyNode((SNode*)pRoot); - return cxt.errCode; + SLogicPlanContext cxt = { .pPlanCxt = pCxt }; + int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, pLogicNode); + if (TSDB_CODE_SUCCESS != code) { + return code; } - *pLogicNode = pRoot; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0b32f2ebeab09ce91605926e24cbfd3b9e69258c..a33cb8725356b50924092f2a165a310c58f921ca 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -287,6 +287,13 @@ static SPhysiNode* createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubp return (SPhysiNode*)pScan; } +static SPhysiNode* createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { + SStreamScanPhysiNode* pTableScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + CHECK_ALLOC(pTableScan, NULL); + CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); + return (SPhysiNode*)pTableScan; +} + static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: @@ -296,7 +303,7 @@ static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpl case SCAN_TYPE_SYSTEM_TABLE: return createSystemTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode); case SCAN_TYPE_STREAM: - break; + return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode); default: break; } @@ -495,11 +502,20 @@ static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC } static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); - CHECK_ALLOC(pExchange, NULL); - CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); - pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; - return (SPhysiNode*)pExchange; + if (pCxt->pPlanCxt->streamQuery) { + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + CHECK_ALLOC(pScan, NULL); + pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets); + CHECK_ALLOC(pScan->pScanCols, (SPhysiNode*)pScan); + CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc), (SPhysiNode*)pScan); + return (SPhysiNode*)pScan; + } else { + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + CHECK_ALLOC(pExchange, NULL); + CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); + pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; + return (SPhysiNode*)pExchange; + } } static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) { @@ -509,6 +525,9 @@ static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* p pInterval->interval = pWindowLogicNode->interval; pInterval->offset = pWindowLogicNode->offset; pInterval->sliding = pWindowLogicNode->sliding; + pInterval->intervalUnit = pWindowLogicNode->intervalUnit; + pInterval->slidingUnit = pWindowLogicNode->slidingUnit; + pInterval->pFill = nodesCloneNode(pWindowLogicNode->pFill); SNodeList* pPrecalcExprs = NULL; @@ -634,7 +653,9 @@ static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLog taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode); - pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); + if (!pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { + pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); + } pSubplan->msgType = TDMT_VND_QUERY; } return pSubplan; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 5c47ee52b325dda7fe14352d86dfc566ff13ff5a..4d7d31879bec6c3d37bb9ee6f49386ec87ced48c 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -90,7 +90,7 @@ static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); - TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo); + TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo*); SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS); return pSubplan; } diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 575a8cc5d0a2f0babf85bf7713c33e569a6ec9bf..90d8f05efc153c0332082d507a88087e8f6d5443 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -17,6 +17,7 @@ #include +#include "cmdnodes.h" #include "parser.h" #include "planInt.h" @@ -56,7 +57,8 @@ protected: const string syntaxTreeStr = toString(query_->pRoot, false); SLogicNode* pLogicPlan = nullptr; - SPlanContext cxt = { .queryId = 1, .acctId = 0, .mgmtEpSet = {0}, .pAstRoot = query_->pRoot }; + SPlanContext cxt = { .queryId = 1, .acctId = 0 }; + setPlanContext(query_, &cxt); code = createLogicPlan(&cxt, &pLogicPlan); if (code != TSDB_CODE_SUCCESS) { cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl; @@ -94,6 +96,15 @@ protected: private: static const int max_err_len = 1024; + void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { + if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { + pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; + pCxt->topicQuery = true; + } else { + pCxt->pAstRoot = pQuery->pRoot; + } + } + void reset() { memset(&cxt_, 0, sizeof(cxt_)); memset(errMagBuf_, 0, max_err_len); @@ -178,5 +189,11 @@ TEST_F(PlannerTest, showTables) { setDatabase("root", "test"); bind("show tables"); +} + +TEST_F(PlannerTest, createTopic) { + setDatabase("root", "test"); + + bind("create topic tp as SELECT * FROM st1"); ASSERT_TRUE(run()); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 44f1c454c90dd97d7c0a80f19b5678f316ba0091..79a59a5ecb758a5efaf1fad552052bf0748b56fb 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -155,6 +155,10 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp .ahandle = (void*)pInfo, .handle = pInfo->msgInfo.handle, .code = 0}; + if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH || + pInfo->msgType == TDMT_VND_QUERY_CONTINUE) { + rpcMsg.persistHandle = 1; + } assert(pInfo->fp != NULL); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 6e2d482c051418847cf8260d2cbb731847e38135..dc8ef4f60c632a1a9946285e1cf5034aa6e76cf2 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -138,7 +138,7 @@ typedef struct SQWorkerMgmt { SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx void *nodeObj; putReqToQueryQFp putToQueueFp; - sendReqToDnodeFp sendReqFp; + sendReqFp sendReqFp; } SQWorkerMgmt; #define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 60eb501dd24c06eedaf7ee968677837cce3551f2..783c103bbc3f4e966d4bcc42152bbcda6e5248ba 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1299,7 +1299,7 @@ _return: } int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, - putReqToQueryQFp fp1, sendReqToDnodeFp fp2) { + putReqToQueryQFp fp1, sendReqFp fp2) { if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp1 || NULL == fp2) { qError("invalid param to init qworker"); QW_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index cc4233bd4739f375585260f0e0c7a50df8e57b81..9282b7bf6d1adae806eba7b262393323b4a2c5dc 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -878,7 +878,7 @@ TEST(seqTest, normalCase) { stubSetPutDataBlock(); stubSetGetDataBlock(); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqToDnodeFp)qwtSendReqToDnode); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqFp)qwtSendReqToDnode); ASSERT_EQ(code, 0); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); @@ -917,7 +917,7 @@ TEST(seqTest, cancelFirst) { stubSetStringToPlan(); stubSetRpcSendResponse(); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqToDnodeFp)qwtSendReqToDnode); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqFp)qwtSendReqToDnode); ASSERT_EQ(code, 0); qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); @@ -963,7 +963,7 @@ TEST(seqTest, randCase) { taosSeedRand(taosGetTimestampSec()); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqToDnodeFp)qwtSendReqToDnode); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqFp)qwtSendReqToDnode); ASSERT_EQ(code, 0); int32_t t = 0; @@ -1034,19 +1034,19 @@ TEST(seqTest, multithreadRand) { taosSeedRand(taosGetTimestampSec()); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqToDnodeFp)qwtSendReqToDnode); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqFp)qwtSendReqToDnode); ASSERT_EQ(code, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5,t6; - pthread_create(&(t1), &thattr, queryThread, mgmt); - pthread_create(&(t2), &thattr, readyThread, NULL); - pthread_create(&(t3), &thattr, fetchThread, NULL); - pthread_create(&(t4), &thattr, dropThread, NULL); - pthread_create(&(t5), &thattr, statusThread, NULL); - pthread_create(&(t6), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5,t6; + taosThreadCreate(&(t1), &thattr, queryThread, mgmt); + taosThreadCreate(&(t2), &thattr, readyThread, NULL); + taosThreadCreate(&(t3), &thattr, fetchThread, NULL); + taosThreadCreate(&(t4), &thattr, dropThread, NULL); + taosThreadCreate(&(t5), &thattr, statusThread, NULL); + taosThreadCreate(&(t6), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1095,7 +1095,7 @@ TEST(rcTest, shortExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqToDnodeFp)qwtSendReqToDnode); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqFp)qwtSendReqToDnode); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1104,13 +1104,13 @@ TEST(rcTest, shortExecshortDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1176,7 +1176,7 @@ TEST(rcTest, longExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqToDnodeFp)qwtSendReqToDnode); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqFp)qwtSendReqToDnode); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 1000000; @@ -1185,13 +1185,13 @@ TEST(rcTest, longExecshortDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1259,7 +1259,7 @@ TEST(rcTest, shortExeclongDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqToDnodeFp)qwtSendReqToDnode); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqFp)qwtSendReqToDnode); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1268,13 +1268,13 @@ TEST(rcTest, shortExeclongDelay) { tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -1340,19 +1340,19 @@ TEST(rcTest, dropTest) { taosSeedRand(taosGetTimestampSec()); - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqToDnodeFp)qwtSendReqToDnode); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, (sendReqFp)qwtSendReqToDnode); ASSERT_EQ(code, 0); tsem_init(&qwtTestQuerySem, 0, 0); tsem_init(&qwtTestFetchSem, 0, 0); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, qwtclientThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, mgmt); - pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + TdThread t1,t2,t3,t4,t5; + taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); + taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); + taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a7aea5a7e5a219867f0409aa471787d5ebbd3055..58b5c8340a2152d3335d411be2d2cd58f0fd0caf 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -792,7 +792,7 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { } SFilterGroup *gp = NULL; - while (gp = (SFilterGroup *)taosArrayPop(right)) { + while ((gp = (SFilterGroup *)taosArrayPop(right)) != NULL) { taosArrayPush(group, gp); } @@ -801,7 +801,7 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { if (taosArrayGetSize(right) <= 0) { SFilterGroup *gp = NULL; - while (gp = (SFilterGroup *)taosArrayPop(left)) { + while ((gp = (SFilterGroup *)taosArrayPop(left)) != NULL) { taosArrayPush(group, gp); } @@ -1813,7 +1813,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { if(type == TSDB_DATA_TYPE_NCHAR && (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)){ char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; - int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); + int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); if (len < 0){ qError("filterInitValFieldData taosUcs4ToMbs error 1"); return TSDB_CODE_QRY_APP_ERROR; @@ -2992,7 +2992,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){ char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); - int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ qError("castConvert1 taosUcs4ToMbs error"); }else{ @@ -3052,7 +3052,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg } else { if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); - int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ qError("castConvert1 taosUcs4ToMbs error"); }else{ @@ -3433,7 +3433,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar varDataCopy(dst, src); continue; } - bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4*)varDataVal(dst), bufSize, &len); if(!ret) { qError("filterConverNcharColumns taosMbsToUcs4 error"); return TSDB_CODE_FAILED; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b8cdda9ed179280b9634fc4c15be622523aacd06..8e7eaa0f8c65818504de1eb507f86376aa7dd077 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -239,6 +239,9 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t break; } + + default: + break; } if (param->num > *rowNum) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 0fe7bf6e369fc3c9e9b2d8658ccb45c513b255f3..17ac9b19fd5574b61beb25ec3198dace2992e5fe 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -316,7 +316,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t tmp = realloc(tmp, bufSize); } - int len = taosUcs4ToMbs(varDataVal(pIn->data), varDataLen(pIn->data), tmp); + int len = taosUcs4ToMbs((TdUcs4*)varDataVal(pIn->data), varDataLen(pIn->data), tmp); if (len < 0){ sclError("castConvert taosUcs4ToMbs error 1"); tfree(tmp); diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 9fba6523b6bbfcb165d5e6acff3b53fc73620fd8..993521da8722afd2363e47d65cc48b51526968e6 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -282,8 +282,10 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - SCH_ERR_RET(schLaunchTasksInFlowCtrlListImpl(pJob, ctrl)); + int32_t code = schLaunchTasksInFlowCtrlListImpl(pJob, ctrl);; + SCH_ERR_RET(code); + return code; // to avoid compiler error } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 189124ef234bc9b6ad26b2c0702df10d47482be9..fe102ed6ed77d16b6fde9d41772a1d2663dc5f7a 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -13,26 +13,21 @@ * along with this program. If not, see . */ +#include "catalog.h" +#include "query.h" #include "schedulerInt.h" #include "tmsg.h" -#include "query.h" -#include "catalog.h" #include "tref.h" SSchedulerMgmt schMgmt = {0}; -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { - return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); -} +FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } -FORCE_INLINE int32_t schReleaseJob(int64_t refId) { - return taosReleaseRef(schMgmt.jobRef, refId); -} +FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } -uint64_t schGenTaskId(void) { - return atomic_add_fetch_64(&schMgmt.taskId, 1); -} +uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } +#if 0 uint64_t schGenUUID(void) { static uint64_t hashId = 0; static int32_t requestSerialId = 0; @@ -54,11 +49,11 @@ uint64_t schGenUUID(void) { uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; } +#endif - -int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel *pLevel) { - pTask->plan = pPlan; - pTask->level = pLevel; +int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) { + pTask->plan = pPlan; + pTask->level = pLevel; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); pTask->taskId = schGenTaskId(); pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); @@ -70,7 +65,7 @@ int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel * return TSDB_CODE_SUCCESS; } -void schFreeTask(SSchTask* pTask) { +void schFreeTask(SSchTask *pTask) { if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } @@ -90,22 +85,21 @@ void schFreeTask(SSchTask* pTask) { } } - static FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { int8_t status = SCH_GET_JOB_STATUS(pJob); if (pStatus) { *pStatus = status; } - return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED - || status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING - || status == JOB_TASK_STATUS_SUCCEED); + return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || + status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING || + status == JOB_TASK_STATUS_SUCCEED); } - int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t lastMsgType = SCH_GET_TASK_LASTMSG_TYPE(pTask); int32_t taskStatus = SCH_GET_TASK_STATUS(pTask); + switch (msgType) { case TDMT_VND_CREATE_TABLE_RSP: case TDMT_VND_SUBMIT_RSP: @@ -114,7 +108,8 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m case TDMT_VND_FETCH_RSP: case TDMT_VND_DROP_TASK: if (lastMsgType != (msgType - 1)) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -126,7 +121,6 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m break; default: SCH_TASK_ELOG("unknown rsp msg, type:%s, status:%s", TMSG_INFO(msgType), jobTaskStatusStr(taskStatus)); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -135,7 +129,6 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m return TSDB_CODE_SUCCESS; } - int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t code = 0; @@ -147,37 +140,34 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { if (oriStatus == newStatus) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_NOT_START) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_NOT_START: if (newStatus != JOB_TASK_STATUS_EXECUTING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_EXECUTING: - if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_CANCELLING - && newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED && newStatus != JOB_TASK_STATUS_FAILED && + newStatus != JOB_TASK_STATUS_CANCELLING && newStatus != JOB_TASK_STATUS_CANCELLED && + newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_PARTIAL_SUCCEED: - if (newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_SUCCEED - && newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_SUCCEED && + newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_SUCCEED: case JOB_TASK_STATUS_FAILED: @@ -185,13 +175,13 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { if (newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_DROPPING: SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); break; - + default: SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); @@ -211,27 +201,25 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { _return: SCH_JOB_ELOG("invalid job status update, from %s to %s", jobTaskStatusStr(oriStatus), jobTaskStatusStr(newStatus)); - SCH_ERR_RET(code); } - int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t i = 0; i < pJob->levelNum; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); - + for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); SSubplan *pPlan = pTask->plan; - int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0; - int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0; + int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0; + int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0; if (childNum > 0) { if (pJob->levelIdx == pLevel->level) { SCH_JOB_ELOG("invalid query plan, lowest level, childNum:%d", childNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + pTask->children = taosArrayInit(childNum, POINTER_BYTES); if (NULL == pTask->children) { SCH_TASK_ELOG("taosArrayInit %d children failed", childNum); @@ -240,7 +228,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < childNum; ++n) { - SSubplan *child = (SSubplan*)nodesListGetNode(pPlan->pChildren, n); + SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); if (NULL == childTask || NULL == *childTask) { SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -258,7 +246,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("invalid task info, level:0, parentNum:%d", parentNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + pTask->parents = taosArrayInit(parentNum, POINTER_BYTES); if (NULL == pTask->parents) { SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum); @@ -272,7 +260,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < parentNum; ++n) { - SSubplan *parent = (SSubplan*)nodesListGetNode(pPlan->pParents, n); + SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -283,7 +271,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - } + } SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); } @@ -298,11 +286,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { return TSDB_CODE_SUCCESS; } - int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { - SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", pTask->candidateIdx, + (int32_t)taosArrayGetSize(pTask->candidateAddrs)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -311,7 +299,6 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr) { if (NULL == taosArrayPush(pTask->execAddrs, addr)) { SCH_TASK_ELOG("taosArrayPush addr to execAddr list failed, errno:%d", errno); @@ -321,23 +308,25 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad return TSDB_CODE_SUCCESS; } - int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { int32_t code = 0; pJob->queryId = pDag->queryId; - + if (pDag->numOfSubplans <= 0) { SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); if (levelNum <= 0) { SCH_JOB_ELOG("invalid level num:%d", levelNum); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SHashObj *planToTask = taosHashInit(SCHEDULE_DEFAULT_MAX_TASK_NUM, taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + SHashObj *planToTask = taosHashInit( + SCHEDULE_DEFAULT_MAX_TASK_NUM, + taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, + HASH_NO_LOCK); if (NULL == planToTask) { SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -354,10 +343,10 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pJob->subPlans = pDag->pSubplans; - SSchLevel level = {0}; + SSchLevel level = {0}; SNodeListNode *plans = NULL; - int32_t taskNum = 0; - SSchLevel *pLevel = NULL; + int32_t taskNum = 0; + SSchLevel *pLevel = NULL; level.status = JOB_TASK_STATUS_NOT_START; @@ -369,8 +358,8 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pLevel = taosArrayGet(pJob->levels, i); pLevel->level = i; - - plans = (SNodeListNode*)nodesListGetNode(pDag->pSubplans, i); + + plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); if (NULL == plans) { SCH_JOB_ELOG("empty level plan, level:%d", i); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); @@ -383,15 +372,15 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { } pLevel->taskNum = taskNum; - + pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask)); if (NULL == pLevel->subTasks) { SCH_JOB_ELOG("taosArrayInit %d failed", taskNum); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + for (int32_t n = 0; n < taskNum; ++n) { - SSubplan *plan = (SSubplan*)nodesListGetNode(plans->pNodeList, n); + SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); SCH_SET_JOB_TYPE(pJob, plan->subplanType); @@ -399,13 +388,13 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchTask *pTask = &task; SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel)); - + void *p = taosArrayPush(pLevel->subTasks, &task); if (NULL == p) { SCH_TASK_ELOG("taosArrayPush task to level failed, level:%d, taskIdx:%d", pLevel->level, n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -454,10 +443,10 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { int32_t nodeNum = 0; if (pJob->nodeList) { nodeNum = taosArrayGetSize(pJob->nodeList); - + for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i); - + if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -472,14 +461,14 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } -/* - for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { - strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); - epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; - - ++epSet->numOfEps; - } -*/ + /* + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { + strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); + epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; + + ++epSet->numOfEps; + } + */ return TSDB_CODE_SUCCESS; } @@ -491,7 +480,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("task already in execTask list, code:%x", code); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -512,11 +501,10 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - SCH_TASK_ELOG("task already in succTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -524,13 +512,13 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to succTask list, numOfTasks:%d", taosHashGetSize(pJob->succTasks)); - + return TSDB_CODE_SUCCESS; } int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = false; - + if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); } @@ -539,11 +527,11 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_WLOG("task already in failTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -551,11 +539,10 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to failTask list, numOfTasks:%d", taosHashGetSize(pJob->failTasks)); - + return TSDB_CODE_SUCCESS; } - int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) { SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); @@ -565,11 +552,11 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_ELOG("task already in execTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -577,11 +564,10 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); - + return TSDB_CODE_SUCCESS; } - int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { int8_t status = 0; ++pTask->tryTimes; @@ -604,7 +590,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo return TSDB_CODE_SUCCESS; } - //TODO CHECK epList/condidateList + // TODO CHECK epList/condidateList if (SCH_IS_DATA_SRC_TASK(pTask)) { if (pTask->tryTimes >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { *needRetry = false; @@ -613,7 +599,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo } } else { int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); - + if ((pTask->candidateIdx + 1) >= candidateNum) { *needRetry = false; SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", pTask->candidateIdx, candidateNum); @@ -647,9 +633,9 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { } int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { - int32_t code = 0; + int32_t code = 0; SSchHbTrans *hb = NULL; - + while (true) { hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); if (NULL == hb) { @@ -663,9 +649,11 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { SCH_ERR_RET(code); } - qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", - trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, trans->trans.transHandle); - + qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 + ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", + trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, + trans->trans.transHandle); + return TSDB_CODE_SUCCESS; } @@ -673,11 +661,11 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { } SCH_LOCK(SCH_WRITE, &hb->lock); - + if (hb->seqId >= trans->seqId) { - qDebug("hb trans seqId is old, seqId:%" PRId64 ", currentId:%" PRId64 ", nodeId:%d, fqdn:%s, port:%d", - trans->seqId, hb->seqId, epId->nodeId, epId->ep.fqdn, epId->ep.port); - + qDebug("hb trans seqId is old, seqId:%" PRId64 ", currentId:%" PRId64 ", nodeId:%d, fqdn:%s, port:%d", trans->seqId, + hb->seqId, epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_UNLOCK(SCH_WRITE, &hb->lock); return TSDB_CODE_SUCCESS; } @@ -687,9 +675,11 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { SCH_UNLOCK(SCH_WRITE, &hb->lock); - qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", - trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, trans->trans.transHandle); - + qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 + ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", + trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, + trans->trans.transHandle); + return TSDB_CODE_SUCCESS; } @@ -736,11 +726,10 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod int32_t code = atomic_load_32(&pJob->errCode); SCH_JOB_DLOG("job failed with error: %s", tstrerror(code)); - + SCH_RET(code); } - // Note: no more task error processing, handled in function internal int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_FAILED, errCode)); @@ -751,18 +740,16 @@ int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_DROPPING, errCode)); } - - // Note: no more task error processing, handled in function internal int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { int32_t code = 0; - + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); if (pJob->attr.syncSchedule) { tsem_post(&pJob->rspSem); } - + if (atomic_load_8(&pJob->userFetch)) { SCH_ERR_JRET(schFetchFromRemote(pJob)); } @@ -782,21 +769,21 @@ int32_t schProcessOnDataFetched(SSchJob *job) { // Note: no more task error processing, handled in function internal int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { int8_t status = 0; - + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("task failed not processed cause of job status, job status:%s", jobTaskStatusStr(status)); SCH_RET(atomic_load_32(&pJob->errCode)); } - bool needRetry = false; - bool moved = false; + bool needRetry = false; + bool moved = false; int32_t taskDone = 0; int32_t code = 0; SCH_TASK_DLOG("taskOnFailure, code:%s", tstrerror(errCode)); - + SCH_ERR_JRET(schTaskCheckSetRetry(pJob, pTask, errCode, &needRetry)); - + if (!needRetry) { SCH_TASK_ELOG("task failed and no more retry, code:%s", tstrerror(errCode)); @@ -824,7 +811,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) } } else { SCH_ERR_JRET(schHandleTaskRetry(pJob, pTask)); - + return TSDB_CODE_SUCCESS; } @@ -835,7 +822,7 @@ _return: // Note: no more task error processing, handled in function internal int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { - bool moved = false; + bool moved = false; int32_t code = 0; SCH_TASK_DLOG("taskOnSuccess, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); @@ -847,17 +834,16 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_JRET(schRecordTaskSucceedNode(pJob, pTask)); SCH_ERR_JRET(schLaunchTasksInFlowCtrlList(pJob, pTask)); - + int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { - int32_t taskDone = 0; - + int32_t taskDone = 0; if (SCH_IS_WAIT_ALL_JOB(pJob)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskSucceed++; taskDone = pTask->level->taskSucceed + pTask->level->taskFailed; SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); - + if (taskDone < pTask->level->taskNum) { SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); return TSDB_CODE_SUCCESS; @@ -877,28 +863,31 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { pJob->fetchTask = pTask; SCH_ERR_JRET(schMoveTaskToExecList(pJob, pTask, &moved)); - + SCH_RET(schProcessOnJobPartialSuccess(pJob)); } -/* - if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { - strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); - job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; + /* + if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { + strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); + job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; - ++job->dataSrcEps.numOfEps; - } -*/ + ++job->dataSrcEps.numOfEps; + } + */ for (int32_t i = 0; i < parentNum; ++i) { SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); - int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); + int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); - SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE, .taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; + SDownstreamSourceNode source = {.type = QUERY_NODE_DOWNSTREAM_SOURCE, + .taskId = pTask->taskId, + .schedId = schMgmt.sId, + .addr = pTask->succeedAddr}; qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); - + if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { SCH_ERR_RET(schLaunchTaskImpl(pJob, par)); } @@ -911,11 +900,10 @@ _return: SCH_RET(schProcessOnJobFailure(pJob, code)); } - // Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; - + if (atomic_val_compare_exchange_32(&pJob->remoteFetch, 0, 1) != 0) { SCH_JOB_ELOG("prior fetching not finished, remoteFetch:%d", atomic_load_32(&pJob->remoteFetch)); return TSDB_CODE_SUCCESS; @@ -932,7 +920,7 @@ int32_t schFetchFromRemote(SSchJob *pJob) { SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); return TSDB_CODE_SUCCESS; - + _return: atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); @@ -940,15 +928,14 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pJob->fetchTask, code)); } - // Note: no more task error processing, handled in function internal -int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { +int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, + int32_t rspCode) { int32_t code = 0; - int8_t status = 0; - + int8_t status = 0; + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), rspCode); - SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -956,43 +943,43 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch switch (msgType) { case TDMT_VND_CREATE_TABLE_RSP: { - SVCreateTbBatchRsp batchRsp = {0}; - if (msg) { - tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp); - if (batchRsp.rspList) { - int32_t num = taosArrayGetSize(batchRsp.rspList); - for (int32_t i = 0; i < num; ++i) { - SVCreateTbRsp *rsp = taosArrayGet(batchRsp.rspList, i); - if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { - taosArrayDestroy(batchRsp.rspList); - SCH_ERR_JRET(rsp->code); - } + SVCreateTbBatchRsp batchRsp = {0}; + if (msg) { + tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp); + if (batchRsp.rspList) { + int32_t num = taosArrayGetSize(batchRsp.rspList); + for (int32_t i = 0; i < num; ++i) { + SVCreateTbRsp *rsp = taosArrayGet(batchRsp.rspList, i); + if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { + taosArrayDestroy(batchRsp.rspList); + SCH_ERR_JRET(rsp->code); } - - taosArrayDestroy(batchRsp.rspList); } - } - - SCH_ERR_JRET(rspCode); - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - - break; - } - case TDMT_VND_SUBMIT_RSP: { - if (msg) { - SSubmitRsp *rsp = (SSubmitRsp *)msg; - SCH_ERR_JRET(rsp->code); - - pJob->resNumOfRows += rsp->affectedRows; + taosArrayDestroy(batchRsp.rspList); } + } + + SCH_ERR_JRET(rspCode); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + break; + } + case TDMT_VND_SUBMIT_RSP: { + if (msg) { + SSubmitRsp *rsp = (SSubmitRsp *)msg; + SCH_ERR_JRET(rsp->code); + } + SCH_ERR_JRET(rspCode); - SCH_ERR_JRET(rspCode); + SSubmitRsp *rsp = (SSubmitRsp *)msg; + if (rsp) { + pJob->resNumOfRows += rsp->affectedRows; + } - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - break; - } + break; + } case TDMT_VND_QUERY_RSP: { SQueryTableRsp rsp = {0}; if (msg) { @@ -1009,52 +996,51 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); break; - } + } case TDMT_VND_RES_READY_RSP: { - SResReadyRsp *rsp = (SResReadyRsp *)msg; - - SCH_ERR_JRET(rspCode); - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - SCH_ERR_JRET(rsp->code); - - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - - break; + SResReadyRsp *rsp = (SResReadyRsp *)msg; + + SCH_ERR_JRET(rspCode); + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - case TDMT_VND_FETCH_RSP: { - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; + SCH_ERR_JRET(rsp->code); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - SCH_ERR_JRET(rspCode); - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - if (pJob->resData) { - SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->resData); - tfree(rsp); - SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); - } + break; + } + case TDMT_VND_FETCH_RSP: { + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; - atomic_store_ptr(&pJob->resData, rsp); - atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); + SCH_ERR_JRET(rspCode); + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } - if (rsp->completed) { - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); - } + if (pJob->resData) { + SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->resData); + tfree(rsp); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); + } - SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + atomic_store_ptr(&pJob->resData, rsp); + atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); - schProcessOnDataFetched(pJob); - break; + if (rsp->completed) { + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); } + + SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + + schProcessOnDataFetched(pJob); + break; + } case TDMT_VND_DROP_TASK_RSP: { - // SHOULD NEVER REACH HERE - SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:%" PRIx64, pJob->refId); - SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); - break; - } + // SHOULD NEVER REACH HERE + SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:%" PRIx64, pJob->refId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + break; + } default: SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%s", msgType, SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1067,15 +1053,15 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } - -int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) { - int32_t code = 0; +int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { + int32_t code = 0; SSchCallbackParam *pParam = (SSchCallbackParam *)param; - SSchTask *pTask = NULL; - + SSchTask *pTask = NULL; + SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { - qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, pParam->queryId, pParam->taskId, pParam->refId); + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, + pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } @@ -1093,8 +1079,8 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in pTask = *task; SCH_TASK_DLOG("rsp msg received, type:%s, code:%s", TMSG_INFO(msgType), tstrerror(rspCode)); - - pTask->handle = pMsg->handle; + + pTask->handle = pMsg->handle; SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: @@ -1107,42 +1093,41 @@ _return: SCH_RET(code); } -int32_t schHandleSubmitCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleSubmitCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_SUBMIT_RSP, code); } -int32_t schHandleCreateTableCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleCreateTableCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_CREATE_TABLE_RSP, code); } -int32_t schHandleQueryCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleQueryCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code); } -int32_t schHandleFetchCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleFetchCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_FETCH_RSP, code); } -int32_t schHandleReadyCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleReadyCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code); } -int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { SSchCallbackParam *pParam = (SSchCallbackParam *)param; - qDebug("QID:%"PRIx64",TID:%"PRIx64" drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); + qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); } - -int32_t schHandleHbCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { if (code) { qError("hb rsp error:%s", tstrerror(code)); SCH_ERR_RET(code); } - + SSchedulerHbRsp rsp = {0}; SSchCallbackParam *pParam = (SSchCallbackParam *)param; - + if (tDeserializeSSchedulerHbRsp(pMsg->pData, pMsg->len, &rsp)) { qError("invalid hb rsp msg, size:%d", pMsg->len); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1153,21 +1138,22 @@ int32_t schHandleHbCallback(void* param, const SDataBuf* pMsg, int32_t code) { trans.seqId = rsp.seqId; trans.trans.transInst = pParam->transport; trans.trans.transHandle = pMsg->handle; - + SCH_RET(schUpdateHbConnection(&rsp.epId, &trans)); } int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i); - + SSchJob *pJob = schAcquireJob(taskStatus->refId); if (NULL == pJob) { - qWarn("job not found, refId:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64, taskStatus->refId, taskStatus->queryId, taskStatus->taskId); - //TODO DROP TASK FROM SERVER!!!! + qWarn("job not found, refId:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64, taskStatus->refId, + taskStatus->queryId, taskStatus->taskId); + // TODO DROP TASK FROM SERVER!!!! continue; } - + // TODO schReleaseJob(taskStatus->refId); @@ -1180,22 +1166,21 @@ _return: SCH_RET(code); } - int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { switch (msgType) { case TDMT_VND_CREATE_TABLE: *fp = schHandleCreateTableCallback; break; - case TDMT_VND_SUBMIT: + case TDMT_VND_SUBMIT: *fp = schHandleSubmitCallback; break; - case TDMT_VND_QUERY: + case TDMT_VND_QUERY: *fp = schHandleQueryCallback; break; - case TDMT_VND_RES_READY: + case TDMT_VND_RES_READY: *fp = schHandleReadyCallback; break; - case TDMT_VND_FETCH: + case TDMT_VND_FETCH: *fp = schHandleFetchCallback; break; case TDMT_VND_DROP_TASK: @@ -1212,13 +1197,13 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { return TSDB_CODE_SUCCESS; } - -int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize) { +int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet *epSet, int32_t msgType, void *msg, + uint32_t msgSize) { int32_t code = 0; SSchTrans *trans = (SSchTrans *)transport; - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1238,15 +1223,14 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* param->taskId = SCH_TASK_ID(pTask); param->transport = trans->transInst; - pMsgSendInfo->param = param; pMsgSendInfo->msgInfo.pData = msg; pMsgSendInfo->msgInfo.len = msgSize; - pMsgSendInfo->msgInfo.handle = trans->transHandle; + pMsgSendInfo->msgInfo.handle = trans->transHandle; pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - - int64_t transporterId = 0; + + int64_t transporterId = 0; code = asyncSendMsgToServer(trans->transInst, epSet, &transporterId, pMsgSendInfo); if (code) { SCH_ERR_JRET(code); @@ -1256,7 +1240,7 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* return TSDB_CODE_SUCCESS; _return: - + tfree(param); tfree(pMsgSendInfo); SCH_RET(code); @@ -1264,9 +1248,9 @@ _return: int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void *msg = NULL; - int32_t code = 0; - bool isCandidateAddr = false; + void *msg = NULL; + int32_t code = 0; + bool isCandidateAddr = false; if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; @@ -1300,13 +1284,13 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SSubQueryMsg *pMsg = msg; pMsg->header.vgId = htonl(addr->nodeId); - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->refId = htobe64(pJob->refId); - pMsg->taskType = TASK_TYPE_TEMP; - pMsg->phyLen = htonl(pTask->msgLen); - pMsg->sqlLen = htonl(len); + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(pJob->queryId); + pMsg->taskId = htobe64(pTask->taskId); + pMsg->refId = htobe64(pJob->refId); + pMsg->taskType = TASK_TYPE_TEMP; + pMsg->phyLen = htonl(pTask->msgLen); + pMsg->sqlLen = htonl(len); memcpy(pMsg->msg, pJob->sql, len); memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); @@ -1323,12 +1307,12 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } SResReadyReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskId = htobe64(pTask->taskId); break; } case TDMT_VND_FETCH: { @@ -1338,32 +1322,32 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + SResFetchReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskId = htobe64(pTask->taskId); break; } - case TDMT_VND_DROP_TASK:{ + case TDMT_VND_DROP_TASK: { msgSize = sizeof(STaskDropReq); msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + STaskDropReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->refId = htobe64(pJob->refId); + pMsg->taskId = htobe64(pTask->taskId); + pMsg->refId = htobe64(pJob->refId); break; } case TDMT_VND_QUERY_HEARTBEAT: { @@ -1403,24 +1387,24 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (isCandidateAddr) { SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); } - + return TSDB_CODE_SUCCESS; _return: SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); - + tfree(msg); SCH_RET(code); } int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SQueryNodeEpId epId = {0}; + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); - + SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); if (NULL == hb) { SCH_ERR_RET(schBuildAndSendMsg(pJob, NULL, addr, TDMT_VND_QUERY_HEARTBEAT)); @@ -1430,11 +1414,11 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { } int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { - int8_t status = 0; + int8_t status = 0; int32_t code = 0; atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); - + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("no need to launch task cause of job status, job status:%s", jobTaskStatusStr(status)); @@ -1449,30 +1433,31 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { SSubplan *plan = pTask->plan; - if (NULL == pTask->msg) { // TODO add more detailed reason for failure + if (NULL == pTask->msg) { // TODO add more detailed reason for failure code = qSubPlanToString(plan, &pTask->msg, &pTask->msgLen); if (TSDB_CODE_SUCCESS != code) { - SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); + SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, + pTask->msgLen); SCH_ERR_RET(code); } else { SCH_TASK_DLOG("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } } - + SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); if (SCH_IS_QUERY_JOB(pJob)) { SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); } - + SCH_ERR_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); - + return TSDB_CODE_SUCCESS; } // Note: no more error processing, handled in function internal int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { - bool enough = false; + bool enough = false; int32_t code = 0; if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -1502,11 +1487,9 @@ int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level) { return TSDB_CODE_SUCCESS; } - - int32_t schLaunchJob(SSchJob *pJob) { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); - + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_EXECUTING)); SCH_ERR_RET(schCheckJobNeedFlowCtrl(pJob, level)); @@ -1523,7 +1506,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { } int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); - + if (size <= 0) { SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; @@ -1549,9 +1532,9 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { SSchTask *pTask = *(SSchTask **)pIter; schDropTaskOnExecutedNode(pJob, pTask); - + pIter = taosHashIterate(list, pIter); - } + } } void schDropJobAllTasks(SSchJob *pJob) { @@ -1561,10 +1544,9 @@ void schDropJobAllTasks(SSchJob *pJob) { } int32_t schCancelJob(SSchJob *pJob) { - //TODO - - //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST + // TODO + // TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST } void schFreeJobImpl(void *job) { @@ -1574,7 +1556,7 @@ void schFreeJobImpl(void *job) { SSchJob *pJob = job; uint64_t queryId = pJob->queryId; - int64_t refId = pJob->refId; + int64_t refId = pJob->refId; if (pJob->status == JOB_TASK_STATUS_EXECUTING) { schCancelJob(pJob); @@ -1582,55 +1564,54 @@ void schFreeJobImpl(void *job) { schDropJobAllTasks(pJob); - pJob->subPlans = NULL; // it is a reference to pDag->pSubplans - + pJob->subPlans = NULL; // it is a reference to pDag->pSubplans + int32_t numOfLevels = taosArrayGetSize(pJob->levels); - for(int32_t i = 0; i < numOfLevels; ++i) { + for (int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); schFreeFlowCtrl(pLevel); - + int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); - for(int32_t j = 0; j < numOfTasks; ++j) { - SSchTask* pTask = taosArrayGet(pLevel->subTasks, j); + for (int32_t j = 0; j < numOfTasks; ++j) { + SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); schFreeTask(pTask); } taosArrayDestroy(pLevel->subTasks); } - + taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->failTasks); taosHashCleanup(pJob->succTasks); - + taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); tfree(pJob->resData); - tfree(pJob); - qDebug("QID:0x%"PRIx64" job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); } - -static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan* pDag, int64_t *job, const char* sql, bool syncSchedule) { - qDebug("QID:0x%"PRIx64" job started", pDag->queryId); +static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, + bool syncSchedule) { + qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { - qDebug("QID:0x%"PRIx64" input exec nodeList is empty", pDag->queryId); + qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pDag->queryId); } - int32_t code = 0; + int32_t code = 0; SSchJob *pJob = calloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("QID:%"PRIx64" calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; - pJob->sql = sql; + pJob->sql = sql; if (pNodeList != NULL) { pJob->nodeList = taosArrayDup(pNodeList); @@ -1638,19 +1619,22 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan* pD SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob)); - pJob->execTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->execTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->execTasks) { SCH_JOB_ELOG("taosHashInit %d execTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->succTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->succTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->succTasks) { SCH_JOB_ELOG("taosHashInit %d succTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->failTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->failTasks = + taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->failTasks) { SCH_JOB_ELOG("taosHashInit %d failTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1670,9 +1654,9 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan* pD SCH_ERR_JRET(schLaunchJob(pJob)); schAcquireJob(pJob->refId); - + *job = pJob->refId; - + if (syncSchedule) { SCH_JOB_DLOG("will wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); tsem_wait(&pJob->rspSem); @@ -1681,7 +1665,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan* pD SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); schReleaseJob(pJob->refId); - + return TSDB_CODE_SUCCESS; _return: @@ -1690,7 +1674,6 @@ _return: SCH_RET(code); } - int32_t schedulerInit(SSchedulerCfg *cfg) { if (schMgmt.jobRef) { qError("scheduler already initialized"); @@ -1699,7 +1682,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { if (cfg) { schMgmt.cfg = *cfg; - + if (schMgmt.cfg.maxJobNum == 0) { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; } @@ -1710,7 +1693,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; schMgmt.cfg.maxNodeTableNum = SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM; } - + schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { qError("init schduler jobRef failed, num:%u", schMgmt.cfg.maxJobNum); @@ -1728,12 +1711,13 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); } - qInfo("scheduler %"PRIx64" initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); - + qInfo("scheduler %" PRIx64 " initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); + return TSDB_CODE_SUCCESS; } -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan* pDag, int64_t *pJob, const char* sql, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, + SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1746,20 +1730,21 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan* pDag, in pRes->numOfRows = job->resNumOfRows; schReleaseJob(*pJob); - + return TSDB_CODE_SUCCESS; } -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan* pDag, const char* sql, int64_t *pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pDag, const char *sql, int64_t *pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, false)); - + return TSDB_CODE_SUCCESS; } +#if 0 int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks) { if (NULL == pDag || pDag->numOfSubplans <= 0 || LIST_LENGTH(pDag->pSubplans) == 0) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1880,14 +1865,14 @@ _return: SCH_RET(code); } +#endif - -int32_t schedulerFetchRows(int64_t job, void** pData) { +int32_t schedulerFetchRows(int64_t job, void **pData) { if (NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t code = 0; + int32_t code = 0; SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); @@ -1936,7 +1921,6 @@ int32_t schedulerFetchRows(int64_t job, void** pData) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } - while (true) { *pData = atomic_load_ptr(&pJob->resData); if (*pData != atomic_val_compare_exchange_ptr(&pJob->resData, *pData, NULL)) { @@ -1961,7 +1945,7 @@ int32_t schedulerFetchRows(int64_t job, void** pData) { _return: atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - + schReleaseJob(job); SCH_RET(code); @@ -2014,17 +1998,17 @@ void schedulerFreeTaskList(SArray *taskList) { taosArrayDestroy(taskList); } - + void schedulerDestroy(void) { if (schMgmt.jobRef) { SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0); - + while (pJob) { taosRemoveRef(schMgmt.jobRef, pJob->refId); - + pJob = taosIterateRef(schMgmt.jobRef, pJob->refId); } - + taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = 0; } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 376ed1e2bc1f5015bd5c8c2466ac77877cf53ce1..b5bd68ba86bf2974be4941def7496c4a629b4cca 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -684,11 +684,11 @@ TEST(queryTest, normalCase) { pIter = taosHashIterate(pJob->execTasks, pIter); } - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, &job); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); void *data = NULL; code = schedulerFetchRows(job, &data); @@ -780,11 +780,11 @@ TEST(queryTest, flowCtrlCase) { } - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, &job); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); void *data = NULL; code = schedulerFetchRows(job, &data); @@ -834,11 +834,11 @@ TEST(insertTest, normalCase) { schtSetPlanToString(); schtSetAsyncSendMsgToServer(); - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1; - pthread_create(&(thread1), &thattr, schtSendRsp, &insertJobRefId); + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", &res); @@ -851,13 +851,13 @@ TEST(insertTest, normalCase) { } TEST(multiThread, forceFree) { - pthread_attr_t thattr; - pthread_attr_init(&thattr); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); - pthread_t thread1, thread2, thread3; - pthread_create(&(thread1), &thattr, schtRunJobThread, NULL); - pthread_create(&(thread2), &thattr, schtFreeJobThread, NULL); - pthread_create(&(thread3), &thattr, schtFetchRspThread, NULL); + TdThread thread1, thread2, thread3; + taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL); + taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL); + taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL); while (true) { if (schtTestDeadLoop) { diff --git a/source/libs/sync/inc/syncCommit.h b/source/libs/sync/inc/syncCommit.h new file mode 100644 index 0000000000000000000000000000000000000000..c76236d5bfd3ca3050c7dbe3900a5644c4388911 --- /dev/null +++ b/source/libs/sync/inc/syncCommit.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_COMMIT_H +#define _TD_LIBS_SYNC_COMMIT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncInt.h" +#include "taosdef.h" + +// \* Leader i advances its commitIndex. +// \* This is done as a separate step from handling AppendEntries responses, +// \* in part to minimize atomic regions, and in part so that leaders of +// \* single-server clusters are able to mark entries committed. +// AdvanceCommitIndex(i) == +// /\ state[i] = Leader +// /\ LET \* The set of servers that agree up through index. +// Agree(index) == {i} \cup {k \in Server : +// matchIndex[i][k] >= index} +// \* The maximum indexes for which a quorum agrees +// agreeIndexes == {index \in 1..Len(log[i]) : +// Agree(index) \in Quorum} +// \* New value for commitIndex'[i] +// newCommitIndex == +// IF /\ agreeIndexes /= {} +// /\ log[i][Max(agreeIndexes)].term = currentTerm[i] +// THEN +// Max(agreeIndexes) +// ELSE +// commitIndex[i] +// IN commitIndex' = [commitIndex EXCEPT ![i] = newCommitIndex] +// /\ UNCHANGED <> +// +void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode); +bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index); +bool syncAgree(SSyncNode* pSyncNode, SyncIndex index); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_COMMIT_H*/ diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 019c291efc03a59e56fae223717718cf0a416de1..85a82dcfb70ffd7431e41c14d81263344ec68ddb 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -39,7 +39,6 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode); - int32_t syncNodeElect(SSyncNode* pSyncNode); int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg); diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 40ff79287b34367d242764fce5794a32b34f8806..66c7c8620df10728ef54c7ccf08c5e854d493f3b 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -31,10 +31,10 @@ extern "C" { #define TIMER_MAX_MS 0x7FFFFFFF #define ENV_TICK_TIMER_MS 1000 #define PING_TIMER_MS 1000 -#define ELECT_TIMER_MS_MIN 150 -#define ELECT_TIMER_MS_MAX 300 +#define ELECT_TIMER_MS_MIN 1500 +#define ELECT_TIMER_MS_MAX 3000 #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 30 +#define HEARTBEAT_TIMER_MS 300 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 352d30c8d72b131b934f430971942fa718d54fd4..0185d5f211669c926f754b3488b3088a0ad35e5f 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -35,7 +35,7 @@ extern "C" { typedef struct SSyncIO { STaosQueue *pMsgQ; STaosQset * pQset; - pthread_t consumerTid; + TdThread consumerTid; void * serverRpc; void * clientRpc; @@ -50,6 +50,7 @@ typedef struct SSyncIO { void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg); + int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg); int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg); int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg); int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 15c719b76e08d27dda223275811f96433550ba84..8e36424f192e03fc72b6878f184ce075ecbdd0bc 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -70,6 +70,9 @@ extern "C" { struct SyncTimeout; typedef struct SyncTimeout SyncTimeout; +struct SyncClientRequest; +typedef struct SyncClientRequest SyncClientRequest; + struct SyncPing; typedef struct SyncPing SyncPing; @@ -185,6 +188,7 @@ typedef struct SSyncNode { // callback int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg); int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg); + int32_t (*FpOnClientRequest)(SSyncNode* ths, SyncClientRequest* pMsg); int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg); int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg); @@ -232,7 +236,6 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode); // raft vote -------------- void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForSelf(SSyncNode* pSyncNode); -void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 5785089a20beb96df31dcb7eb6a003d607f68d8e..ffb2ae3a26e3e356185e41ffc606d2672b6af320 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -163,15 +163,15 @@ typedef struct SyncClientRequest { } SyncClientRequest; SyncClientRequest* syncClientRequestBuild(uint32_t dataLen); -SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak); +SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak); // step 1 void syncClientRequestDestroy(SyncClientRequest* pMsg); void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len); SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len); -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); +void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); +SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3 cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); char* syncClientRequest2Str(const SyncClientRequest* pMsg); diff --git a/source/libs/sync/inc/syncOnMessage.h b/source/libs/sync/inc/syncOnMessage.h index 7cb186a8121800134fad6d1896870e85cbe503b1..2f8856e652a84332a9455fb1f6bc26bf8a975e89 100644 --- a/source/libs/sync/inc/syncOnMessage.h +++ b/source/libs/sync/inc/syncOnMessage.h @@ -25,6 +25,46 @@ extern "C" { #include #include "taosdef.h" +// TLA+ Spec +// Receive(m) == +// LET i == m.mdest +// j == m.msource +// IN \* Any RPC with a newer term causes the recipient to advance +// \* its term first. Responses with stale terms are ignored. +// \/ UpdateTerm(i, j, m) +// \/ /\ m.mtype = RequestVoteRequest +// /\ HandleRequestVoteRequest(i, j, m) +// \/ /\ m.mtype = RequestVoteResponse +// /\ \/ DropStaleResponse(i, j, m) +// \/ HandleRequestVoteResponse(i, j, m) +// \/ /\ m.mtype = AppendEntriesRequest +// /\ HandleAppendEntriesRequest(i, j, m) +// \/ /\ m.mtype = AppendEntriesResponse +// /\ \/ DropStaleResponse(i, j, m) +// \/ HandleAppendEntriesResponse(i, j, m) + +// DuplicateMessage(m) == +// /\ Send(m) +// /\ UNCHANGED <> + +// DropMessage(m) == +// /\ Discard(m) +// /\ UNCHANGED <> + +// Next == /\ \/ \E i \in Server : Restart(i) +// \/ \E i \in Server : Timeout(i) +// \/ \E i,j \in Server : RequestVote(i, j) +// \/ \E i \in Server : BecomeLeader(i) +// \/ \E i \in Server, v \in Value : ClientRequest(i, v) +// \/ \E i \in Server : AdvanceCommitIndex(i) +// \/ \E i,j \in Server : AppendEntries(i, j) +// \/ \E m \in DOMAIN messages : Receive(m) +// \/ \E m \in DOMAIN messages : DuplicateMessage(m) +// \/ \E m \in DOMAIN messages : DropMessage(m) +// \* History variable that tracks every log ever: +// /\ allLogs' = allLogs \cup {log[i] : i \in Server} +// + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 6ba27b0d8a3812e97d151bbcf10a47a0ce19ed4c..037a49f45c5e7a43f77d2dfff13edf7be56398fd 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -40,12 +40,13 @@ typedef struct SSyncRaftEntry { } SSyncRaftEntry; SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); -SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 void syncEntryDestory(SSyncRaftEntry* pEntry); -char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); -SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); +char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 +SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); char* syncEntry2Str(const SSyncRaftEntry* pEntry); +void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 // for debug ---------------------- void syncEntryPrint(const SSyncRaftEntry* pObj); diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 30f7c5d9f7d66b37fec79ab370961ea7b0dc8998..355a08ac8458fc69f9e5296086e66387dafe6026 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -43,11 +43,14 @@ int32_t raftStorePersist(SRaftStore *pRaftStore); int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); -bool raftStoreHasVoted(SRaftStore *pRaftStore); -void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId); -void raftStoreClearVote(SRaftStore *pRaftStore); -void raftStoreNextTerm(SRaftStore *pRaftStore); -void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); +bool raftStoreHasVoted(SRaftStore *pRaftStore); +void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId); +void raftStoreClearVote(SRaftStore *pRaftStore); +void raftStoreNextTerm(SRaftStore *pRaftStore); +void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); +int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson); +cJSON * raftStore2Json(SRaftStore *pRaftStore); +char * raftStore2Str(SRaftStore *pRaftStore); // for debug ------------------- void raftStorePrint(SRaftStore *pObj); diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index aca6205b9dfd75d044ceb8156fb3683af45f097b..6fe18dae384551846cc0eff8ba5b33b413a16e19 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -52,7 +52,6 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode); - int32_t syncNodeReplicate(SSyncNode* pSyncNode); int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 87d6669f59c33a482af03aee58e793a7c30a4f38..e4df93ca47c013348289f057a8a10063e6e6edfe 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -87,13 +87,17 @@ // int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; - syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesCb== term:%lu", ths->pRaftStore->currentTerm); + syncAppendEntriesLog2(logBuf, pMsg); if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); } assert(pMsg->term <= ths->pRaftStore->currentTerm); + // reset elect timer if (pMsg->term == ths->pRaftStore->currentTerm) { ths->leaderCache = pMsg->srcId; syncNodeResetElectTimer(ths); @@ -101,8 +105,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { assert(pMsg->dataLen >= 0); SyncTerm localPreLogTerm = 0; - if (pMsg->prevLogTerm >= SYNC_INDEX_BEGIN && pMsg->prevLogTerm <= ths->pLogStore->getLastIndex(ths->pLogStore)) { - SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm); + if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogIndex); assert(pEntry != NULL); localPreLogTerm = pEntry->term; syncEntryDestory(pEntry); @@ -111,9 +115,9 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { bool logOK = (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && - (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogIndex == localPreLogTerm)); + (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm)); - // reject + // reject request if ((pMsg->term < ths->pRaftStore->currentTerm) || ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); @@ -134,36 +138,118 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // return to follower state if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { syncNodeBecomeFollower(ths); + + // ret or reply? + return ret; } // accept request if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { - bool matchSuccess = false; + bool preMatch = false; if (pMsg->prevLogIndex == SYNC_INDEX_INVALID && ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) { - matchSuccess = true; + preMatch = true; } if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { - SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm); - assert(pEntry != NULL); - if (pMsg->prevLogTerm == pEntry->term) { - matchSuccess = true; + SSyncRaftEntry* pPreEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogIndex); + assert(pPreEntry != NULL); + if (pMsg->prevLogTerm == pPreEntry->term) { + preMatch = true; } - syncEntryDestory(pEntry); + syncEntryDestory(pPreEntry); } - if (matchSuccess) { - // delete conflict entries - if (ths->pLogStore->getLastIndex(ths->pLogStore) > pMsg->prevLogIndex) { - SyncIndex fromIndex = pMsg->prevLogIndex + 1; - ths->pLogStore->truncate(ths->pLogStore, fromIndex); - } + if (preMatch) { + // must has preIndex in local log + assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + + bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); + bool hasAppendEntries = pMsg->dataLen > 0; + + if (hasExtraEntries && hasAppendEntries) { + // conflict + bool conflict = false; + + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = logStoreGetEntry(ths->pLogStore, extraIndex); + assert(pExtraEntry != NULL); + + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); + + assert(extraIndex == pAppendEntry->index); + if (pExtraEntry->term == pAppendEntry->term) { + conflict = true; + } + + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; + + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); + assert(pRollBackEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0); + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pRollBackEntry); + } + } + + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0); + } + } + rpcFreeCont(rpcMsg.pCont); + } + + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); - // append one entry - if (pMsg->dataLen > 0) { - SSyncRaftEntry* pEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - ths->pLogStore->appendEntry(ths->pLogStore, pEntry); - syncEntryDestory(pEntry); + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0); + } + } + rpcFreeCont(rpcMsg.pCont); + + // free memory + syncEntryDestory(pAppendEntry); + + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else { + assert(0); } SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); @@ -171,13 +257,19 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->destId = pMsg->srcId; pReply->term = ths->pRaftStore->currentTerm; pReply->success = true; - pReply->matchIndex = pMsg->prevLogIndex + 1; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; + } SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); syncAppendEntriesReplyDestroy(pReply); + } else { SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); pReply->srcId = ths->myRaftId; @@ -192,11 +284,38 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { syncAppendEntriesReplyDestroy(pReply); } + // maybe update commit index from leader if (pMsg->commitIndex > ths->commitIndex) { + // has commit entry in local if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { - // commit + SyncIndex beginIndex = ths->commitIndex + 1; + SyncIndex endIndex = pMsg->commitIndex; + + // update commit index ths->commitIndex = pMsg->commitIndex; + + // call back Wal ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + + // execute fsm + if (ths->pFsm != NULL) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + if (i != SYNC_INDEX_INVALID) { + SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, i); + assert(pEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (ths->pFsm->FpCommitCb != NULL) { + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + } + + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pEntry); + } + } + } } } } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 61eb4884e2b8e5013adcc6012dec72257d98eac9..817974fd268fa4105c45d51c5b591a79167971dc 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -14,6 +14,7 @@ */ #include "syncAppendEntriesReply.h" +#include "syncCommit.h" #include "syncIndexMgr.h" #include "syncInt.h" #include "syncRaftLog.h" @@ -36,10 +37,14 @@ // int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; - syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplyCb== term:%lu", ths->pRaftStore->currentTerm); + syncAppendEntriesReplyLog2(logBuf, pMsg); if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("DropStaleResponse, receive term:%lu, current term:%lu", pMsg->term, ths->pRaftStore->currentTerm); + sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, + ths->pRaftStore->currentTerm); return ret; } @@ -51,17 +56,19 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p assert(pMsg->term == ths->pRaftStore->currentTerm); if (pMsg->success) { - // nextIndex = reply.matchIndex + 1 + // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); - // matchIndex = reply.matchIndex + // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); // maybe commit - syncNodeMaybeAdvanceCommitIndex(ths); + syncMaybeAdvanceCommitIndex(ths); } else { SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + + // notice! int64, uint64 if (nextIndex > SYNC_INDEX_BEGIN) { --nextIndex; } else { diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c new file mode 100644 index 0000000000000000000000000000000000000000..0d4df8e6cf43aecce88b71145f1f0f4c1fe49744 --- /dev/null +++ b/source/libs/sync/src/syncCommit.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncCommit.h" +#include "syncIndexMgr.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +// \* Leader i advances its commitIndex. +// \* This is done as a separate step from handling AppendEntries responses, +// \* in part to minimize atomic regions, and in part so that leaders of +// \* single-server clusters are able to mark entries committed. +// AdvanceCommitIndex(i) == +// /\ state[i] = Leader +// /\ LET \* The set of servers that agree up through index. +// Agree(index) == {i} \cup {k \in Server : +// matchIndex[i][k] >= index} +// \* The maximum indexes for which a quorum agrees +// agreeIndexes == {index \in 1..Len(log[i]) : +// Agree(index) \in Quorum} +// \* New value for commitIndex'[i] +// newCommitIndex == +// IF /\ agreeIndexes /= {} +// /\ log[i][Max(agreeIndexes)].term = currentTerm[i] +// THEN +// Max(agreeIndexes) +// ELSE +// commitIndex[i] +// IN commitIndex' = [commitIndex EXCEPT ![i] = newCommitIndex] +// /\ UNCHANGED <> +// +void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { + syncIndexMgrLog2("==syncNodeMaybeAdvanceCommitIndex== pNextIndex", pSyncNode->pNextIndex); + syncIndexMgrLog2("==syncNodeMaybeAdvanceCommitIndex== pMatchIndex", pSyncNode->pMatchIndex); + + // update commit index + SyncIndex newCommitIndex = pSyncNode->commitIndex; + for (SyncIndex index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); index > pSyncNode->commitIndex; + --index) { + bool agree = syncAgree(pSyncNode, index); + sTrace("syncMaybeAdvanceCommitIndex syncAgree:%d, index:%ld, pSyncNode->commitIndex:%ld", agree, index, + pSyncNode->commitIndex); + if (agree) { + // term + SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); + assert(pEntry != NULL); + + // cannot commit, even if quorum agree. need check term! + if (pEntry->term == pSyncNode->pRaftStore->currentTerm) { + // update commit index + newCommitIndex = index; + break; + } + } + } + + if (newCommitIndex > pSyncNode->commitIndex) { + SyncIndex beginIndex = pSyncNode->commitIndex + 1; + SyncIndex endIndex = newCommitIndex; + + sTrace("syncMaybeAdvanceCommitIndex sync commit %ld", newCommitIndex); + + // update commit index + pSyncNode->commitIndex = newCommitIndex; + + // call back Wal + pSyncNode->pLogStore->updateCommitIndex(pSyncNode->pLogStore, pSyncNode->commitIndex); + + // execute fsm + if (pSyncNode->pFsm != NULL) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + if (i != SYNC_INDEX_INVALID) { + SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, i); + assert(pEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (pSyncNode->pFsm->FpCommitCb != NULL) { + pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + } + + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pEntry); + } + } + } + } +} + +bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index) { + // I am leader, I agree + if (syncUtilSameId(pRaftId, &(pSyncNode->myRaftId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + return true; + } + + // follower agree + SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, pRaftId); + if (matchIndex >= index) { + return true; + } + + // not agree + return false; +} + +bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) { + int agreeCount = 0; + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + if (syncAgreeIndex(pSyncNode, &(pSyncNode->replicasId[i]), index)) { + ++agreeCount; + } + if (agreeCount >= pSyncNode->quorum) { + return true; + } + } + return false; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 77c3d076988762713a2ffe354c1e91ca54b2e8bd..6ae70689ef50b61b1b48b710ae7028fc526446a6 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -50,6 +50,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { } int32_t syncNodeElect(SSyncNode* pSyncNode) { + int32_t ret = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { syncNodeFollower2Candidate(pSyncNode); } @@ -62,7 +63,15 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->pRaftStore->currentTerm); syncNodeVoteForSelf(pSyncNode); - int32_t ret = syncNodeRequestVotePeers(pSyncNode); + if (voteGrantedMajority(pSyncNode->pVotesGranted)) { + // only myself, to leader + assert(!pSyncNode->pVotesGranted->toLeader); + syncNodeCandidate2Leader(pSyncNode); + pSyncNode->pVotesGranted->toLeader = true; + return ret; + } + + ret = syncNodeRequestVotePeers(pSyncNode); assert(ret == 0); syncNodeResetElectTimer(pSyncNode); diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index dd7161800dfe1d3f780383dc9bd3787b2e2c2623..d47a525e2584652a700b2a9d4da2721fa0571dc2 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -54,27 +54,29 @@ static void syncEnvTick(void *param, void *tmrId) { SSyncEnv *pSyncEnv = (SSyncEnv *)param; if (atomic_load_64(&pSyncEnv->envTickTimerLogicClockUser) <= atomic_load_64(&pSyncEnv->envTickTimerLogicClock)) { ++(pSyncEnv->envTickTimerCounter); - sTrace( - "syncEnvTick do ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " - "envTickTimerMS:%d, tmrId:%p", - pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, - pSyncEnv->envTickTimerMS, tmrId); + sTrace("syncEnvTick do ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 + ", envTickTimerCounter:%" PRIu64 + ", " + "envTickTimerMS:%d, tmrId:%p", + pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, + pSyncEnv->envTickTimerMS, tmrId); // do something, tick ... taosTmrReset(syncEnvTick, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, &pSyncEnv->pEnvTickTimer); } else { - sTrace( - "syncEnvTick pass ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " - "envTickTimerMS:%d, tmrId:%p", - pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, - pSyncEnv->envTickTimerMS, tmrId); + sTrace("syncEnvTick pass ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 + ", envTickTimerCounter:%" PRIu64 + ", " + "envTickTimerMS:%d, tmrId:%p", + pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, + pSyncEnv->envTickTimerMS, tmrId); } } static SSyncEnv *doSyncEnvStart() { SSyncEnv *pSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); assert(pSyncEnv != NULL); - memset(pSyncEnv, 0, sizeof(pSyncEnv)); + memset(pSyncEnv, 0, sizeof(SSyncEnv)); pSyncEnv->envTickTimerCounter = 0; pSyncEnv->envTickTimerMS = ENV_TICK_TIMER_MS; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index c307ec5068f5cd3a74fdba70a4a1f80b5a55980f..19121632e20f55e1402898f352533cf9565c0249 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -211,7 +211,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { // start consumer thread { - if (pthread_create(&io->consumerTid, NULL, syncIOConsumerFunc, io) != 0) { + if (taosThreadCreate(&io->consumerTid, NULL, syncIOConsumerFunc, io) != 0) { sError("failed to create sync consumer thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -228,7 +228,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { static int32_t syncIOStopInternal(SSyncIO *io) { int32_t ret = 0; atomic_store_8(&io->isStart, 0); - pthread_join(io->consumerTid, NULL); + taosThreadJoin(io->consumerTid, NULL); taosTmrCleanUp(io->timerMgr); return ret; } @@ -269,13 +269,23 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { if (io->FpOnSyncPingReply != NULL) { SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } + } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) { + if (io->FpOnSyncClientRequest != NULL) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); + } + } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) { if (io->FpOnSyncRequestVote != NULL) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); io->FpOnSyncRequestVote(io->pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); } @@ -283,6 +293,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) { if (io->FpOnSyncRequestVoteReply != NULL) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); io->FpOnSyncRequestVoteReply(io->pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); } @@ -290,6 +301,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) { if (io->FpOnSyncAppendEntries != NULL) { SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); io->FpOnSyncAppendEntries(io->pSyncNode, pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); } @@ -297,6 +309,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) { if (io->FpOnSyncAppendEntriesReply != NULL) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); io->FpOnSyncAppendEntriesReply(io->pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); } @@ -304,6 +317,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == SYNC_TIMEOUT) { if (io->FpOnSyncTimeout != NULL) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); io->FpOnSyncTimeout(io->pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 95679381978e8c7e574858e87f7e0cf5f93895e4..aa97104180c37dfb2455a79cfef8696717ac5ec1 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -70,22 +70,24 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char u64buf[128]; cJSON *pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncIndexMgr->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); - } - int respondNum = 0; - int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum); - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - arr[i] = pSyncIndexMgr->index[i]; + if (pSyncIndexMgr != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncIndexMgr->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); + } + int respondNum = 0; + int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + arr[i] = pSyncIndexMgr->index[i]; + } + cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); + free(arr); + cJSON_AddItemToObject(pRoot, "index", pIndex); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); } - cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); - free(arr); - cJSON_AddItemToObject(pRoot, "index", pIndex); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); cJSON *pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "pSyncIndexMgr", pRoot); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index aaf6535f402fd1757433ca9253a5b04b28f9da0b..829526512b4403bf88716910902faf2c6bfb3464 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -17,6 +17,7 @@ #include "sync.h" #include "syncAppendEntries.h" #include "syncAppendEntriesReply.h" +#include "syncCommit.h" #include "syncElection.h" #include "syncEnv.h" #include "syncIndexMgr.h" @@ -42,6 +43,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); // on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); // --------------------------------- int32_t syncInit() { @@ -58,11 +60,14 @@ int64_t syncStart(const SSyncInfo* pSyncInfo) { int32_t ret = 0; SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); assert(pSyncNode != NULL); + + // todo : return ref id return ret; } void syncStop(int64_t rid) { - SSyncNode* pSyncNode = NULL; // get pointer from rid + // todo : get pointer from rid + SSyncNode* pSyncNode = NULL; syncNodeClose(pSyncNode); } @@ -72,8 +77,10 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { } int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { - int32_t ret = 0; - SSyncNode* pSyncNode = NULL; // get pointer from rid + int32_t ret = 0; + + // todo : get pointer from rid + SSyncNode* pSyncNode = NULL; if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak); SRpcMsg rpcMsg; @@ -84,13 +91,14 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { } else { sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state)); - ret = -1; // need define err code !! + ret = -1; // todo : need define err code !! } return ret; } ESyncState syncGetMyRole(int64_t rid) { - SSyncNode* pSyncNode = NULL; // get pointer from rid + // todo : get pointer from rid + SSyncNode* pSyncNode = NULL; return pSyncNode->state; } @@ -102,6 +110,12 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); + if (taosMkDir(pSyncInfo->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr()); + return NULL; + } + // init by SSyncInfo pSyncNode->vgId = pSyncInfo->vgId; pSyncNode->syncCfg = pSyncInfo->syncCfg; @@ -143,6 +157,30 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init life cycle + // TLA+ Spec + // InitHistoryVars == /\ elections = {} + // /\ allLogs = {} + // /\ voterLog = [i \in Server |-> [j \in {} |-> <<>>]] + // InitServerVars == /\ currentTerm = [i \in Server |-> 1] + // /\ state = [i \in Server |-> Follower] + // /\ votedFor = [i \in Server |-> Nil] + // InitCandidateVars == /\ votesResponded = [i \in Server |-> {}] + // /\ votesGranted = [i \in Server |-> {}] + // \* The values nextIndex[i][i] and matchIndex[i][i] are never read, since the + // \* leader does not send itself messages. It's still easier to include these + // \* in the functions. + // InitLeaderVars == /\ nextIndex = [i \in Server |-> [j \in Server |-> 1]] + // /\ matchIndex = [i \in Server |-> [j \in Server |-> 0]] + // InitLogVars == /\ log = [i \in Server |-> << >>] + // /\ commitIndex = [i \in Server |-> 0] + // Init == /\ messages = [m \in {} |-> 0] + // /\ InitHistoryVars + // /\ InitServerVars + // /\ InitCandidateVars + // /\ InitLeaderVars + // /\ InitLogVars + // + // init TLA+ server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath); @@ -163,7 +201,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init TLA+ log vars pSyncNode->pLogStore = logStoreCreate(pSyncNode); assert(pSyncNode->pLogStore != NULL); - pSyncNode->commitIndex = 0; + pSyncNode->commitIndex = SYNC_INDEX_INVALID; // init ping timer pSyncNode->pPingTimer = NULL; @@ -192,12 +230,16 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init callback pSyncNode->FpOnPing = syncNodeOnPingCb; pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb; + pSyncNode->FpOnClientRequest = syncNodeOnClientRequestCb; pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; pSyncNode->FpOnTimeout = syncNodeOnTimeoutCb; + // start raft + syncNodeBecomeFollower(pSyncNode); + return pSyncNode; } @@ -353,129 +395,131 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - // init by SSyncInfo - cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); - cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->rpcClient); - cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpSendMsg); - cJSON_AddStringToObject(pRoot, "FpSendMsg", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->queue); - cJSON_AddStringToObject(pRoot, "queue", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpEqMsg); - cJSON_AddStringToObject(pRoot, "FpEqMsg", u64buf); + if (pSyncNode != NULL) { + // init by SSyncInfo + cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); + cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->rpcClient); + cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpSendMsg); + cJSON_AddStringToObject(pRoot, "FpSendMsg", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->queue); + cJSON_AddStringToObject(pRoot, "queue", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpEqMsg); + cJSON_AddStringToObject(pRoot, "FpEqMsg", u64buf); + + // init internal + cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); + cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); + cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); + cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); + + cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); + cJSON* pPeers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); + for (int i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); + } + cJSON* pPeersId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersId", pPeersId); + for (int i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); + } - // init internal - cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); - cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); - cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); - cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); - - cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); - cJSON* pPeers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); - for (int i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); - } - cJSON* pPeersId = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peersId", pPeersId); - for (int i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); - } + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); + cJSON* pReplicasId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); + } - cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); - cJSON* pReplicasId = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); - for (int i = 0; i < pSyncNode->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); + // raft algorithm + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); + cJSON_AddStringToObject(pRoot, "pFsm", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); + cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); + cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); + + // tla+ server vars + cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); + cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); + char tmpBuf[RAFT_STORE_BLOCK_SIZE]; + raftStoreSerialize(pSyncNode->pRaftStore, tmpBuf, sizeof(tmpBuf)); + cJSON_AddStringToObject(pRoot, "pRaftStore", tmpBuf); + + // tla+ candidate vars + cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); + cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); + + // tla+ leader vars + cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); + cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); + + // tla+ log vars + cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRId64 "", pSyncNode->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + + // ping timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); + cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->pingTimerLogicClock); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->pingTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); + cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->pingTimerCounter); + cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); + + // elect timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); + cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->electTimerLogicClock); + cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->electTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); + cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->electTimerCounter); + cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); + + // heartbeat timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); + cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->heartbeatTimerLogicClock); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->heartbeatTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); + cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->heartbeatTimerCounter); + cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); + + // callback + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); + cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); + cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); + cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); + cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); + cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); } - // raft algorithm - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); - cJSON_AddStringToObject(pRoot, "pFsm", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); - cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); - cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); - - // tla+ server vars - cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); - cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); - char tmpBuf[RAFT_STORE_BLOCK_SIZE]; - raftStoreSerialize(pSyncNode->pRaftStore, tmpBuf, sizeof(tmpBuf)); - cJSON_AddStringToObject(pRoot, "pRaftStore", tmpBuf); - - // tla+ candidate vars - cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); - cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); - - // tla+ leader vars - cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); - cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); - - // tla+ log vars - cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); - snprintf(u64buf, sizeof(u64buf), "%ld", pSyncNode->commitIndex); - cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); - - // ping timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); - cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClock); - cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); - cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerCounter); - cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); - - // elect timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); - cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClock); - cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); - cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerCounter); - cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); - - // heartbeat timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); - cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClock); - cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); - cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerCounter); - cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); - - // callback - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); - cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); - cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); - cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); - cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); - cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); - cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); - cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); - cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); return pJson; @@ -498,15 +542,17 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { } void syncNodeBecomeFollower(SSyncNode* pSyncNode) { + // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { pSyncNode->leaderCache = EMPTY_RAFT_ID; } + // state change pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncNodeStopHeartbeatTimer(pSyncNode); - int32_t electMS = syncUtilElectRandomMS(); - syncNodeRestartElectTimer(pSyncNode, electMS); + // reset elect timer + syncNodeResetElectTimer(pSyncNode); } // TLA+ Spec @@ -528,20 +574,32 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode) { // /\ UNCHANGED <> // void syncNodeBecomeLeader(SSyncNode* pSyncNode) { + // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; + + // set leader cache pSyncNode->leaderCache = pSyncNode->myRaftId; for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { + // maybe overwrite myself, no harm + // just do it! pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; } for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { + // maybe overwrite myself, no harm + // just do it! pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; } + // stop elect timer syncNodeStopElectTimer(pSyncNode); - syncNodeStartHeartbeatTimer(pSyncNode); + + // start replicate right now! syncNodeReplicate(pSyncNode); + + // start heartbeat timer + syncNodeStartHeartbeatTimer(pSyncNode); } void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { @@ -566,6 +624,9 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { } // raft vote -------------- + +// just called by syncNodeVoteForSelf +// need assert void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { assert(term == pSyncNode->pRaftStore->currentTerm); assert(!raftStoreHasVoted(pSyncNode->pRaftStore)); @@ -573,6 +634,7 @@ void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) raftStoreVote(pSyncNode->pRaftStore, pRaftId); } +// simulate get vote from outside void syncNodeVoteForSelf(SSyncNode* pSyncNode) { syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId)); @@ -587,8 +649,6 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { syncRequestVoteReplyDestroy(pMsg); } -void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {} - // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); @@ -632,7 +692,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { - sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu", + sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%" PRIu64 ", pingTimerLogicClockUser:%" PRIu64 "", pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } } @@ -653,7 +713,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { - sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%lu, electTimerLogicClockUser:%lu", + sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%" PRIu64 ", electTimerLogicClockUser:%" PRIu64 "", pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); } } @@ -674,7 +734,8 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); } else { - sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu", + sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 + "", pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } } @@ -696,3 +757,59 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg); return ret; } + +// TLA+ Spec +// ClientRequest(i, v) == +// /\ state[i] = Leader +// /\ LET entry == [term |-> currentTerm[i], +// value |-> v] +// newLog == Append(log[i], entry) +// IN log' = [log EXCEPT ![i] = newLog] +// /\ UNCHANGED <> +// +static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { + int32_t ret = 0; + syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); + + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index); + assert(pEntry != NULL); + + if (ths->state == TAOS_SYNC_STATE_LEADER) { + ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + + // start replicate right now! + syncNodeReplicate(ths); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + } + } + rpcFreeCont(rpcMsg.pCont); + + // only myself, maybe commit + syncMaybeAdvanceCommitIndex(ths); + + } else { + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -2); + } + } + rpcFreeCont(rpcMsg.pCont); + } + + syncEntryDestory(pEntry); + return ret; +} diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 509ede274b8d0312ed2d1aee7454d02365a4dd69..3ac4a7a1c083485fe776a92a59c2989b111966a7 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -76,7 +76,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { free(s); } else { - pRoot = syncRpcUnknownMsg2Json(); + pRoot = cJSON_CreateObject(); char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont", s); @@ -212,17 +212,19 @@ SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg) { } cJSON* syncTimeout2Json(const SyncTimeout* pMsg) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - cJSON_AddNumberToObject(pRoot, "timeoutType", pMsg->timeoutType); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->logicClock); - cJSON_AddStringToObject(pRoot, "logicClock", u64buf); - cJSON_AddNumberToObject(pRoot, "timerMS", pMsg->timerMS); - snprintf(u64buf, sizeof(u64buf), "%p", pMsg->data); - cJSON_AddStringToObject(pRoot, "data", u64buf); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + cJSON_AddNumberToObject(pRoot, "timeoutType", pMsg->timeoutType); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->logicClock); + cJSON_AddStringToObject(pRoot, "logicClock", u64buf); + cJSON_AddNumberToObject(pRoot, "timerMS", pMsg->timerMS); + snprintf(u64buf, sizeof(u64buf), "%p", pMsg->data); + cJSON_AddStringToObject(pRoot, "data", u64buf); + } cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncTimeout", pRoot); @@ -239,7 +241,7 @@ char* syncTimeout2Str(const SyncTimeout* pMsg) { // for debug ---------------------- void syncTimeoutPrint(const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); - printf("syncTimeoutPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncTimeoutPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); free(serialized); } @@ -342,50 +344,52 @@ SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg) { } cJSON* syncPing2Json(const SyncPing* pMsg) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + free(s); + s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + free(s); } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - char* s; - s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - free(s); - s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - free(s); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncPing", pRoot); @@ -505,50 +509,52 @@ SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { } cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + free(s); + s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + free(s); } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - char* s; - s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - free(s); - s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - free(s); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncPingReply", pRoot); @@ -565,27 +571,27 @@ char* syncPingReply2Str(const SyncPingReply* pMsg) { // for debug ---------------------- void syncPingReplyPrint(const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); - printf("syncPingReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncPingReplyPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); free(serialized); } void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); - printf("syncPingReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncPingReplyPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); free(serialized); } void syncPingReplyLog(const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); - sTrace("syncPingReplyLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncPingReplyLog | len:%zu | %s", strlen(serialized), serialized); free(serialized); } void syncPingReplyLog2(char* s, const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); - sTrace("syncPingReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); free(serialized); } @@ -602,6 +608,7 @@ SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { return pMsg; } +// step 1. original SRpcMsg => SyncClientRequest, add seqNum, isWeak SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak) { SyncClientRequest* pMsg = syncClientRequestBuild(pOriginalRpcMsg->contLen); pMsg->originalRpcType = pOriginalRpcMsg->msgType; @@ -646,6 +653,7 @@ SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) return pMsg; } +// step 2. SyncClientRequest => RpcMsg, to queue void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg) { memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pMsg->msgType; @@ -658,30 +666,33 @@ void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg syncClientRequestDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); } +// step 3. RpcMsg => SyncClientRequest, from queue SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncClientRequest* pMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); return pMsg; } cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->seqNum); - cJSON_AddStringToObject(pRoot, "seqNum", u64buf); - cJSON_AddNumberToObject(pRoot, "isWeak", pMsg->isWeak); - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - - char* s; - s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - free(s); - s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->seqNum); + cJSON_AddStringToObject(pRoot, "seqNum", u64buf); + cJSON_AddNumberToObject(pRoot, "isWeak", pMsg->isWeak); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + + char* s; + s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + free(s); + s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + free(s); + } cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncClientRequest", pRoot); @@ -785,47 +796,49 @@ SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg) { } cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); - cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); - cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf); + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); + cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); + cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf); + } cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncRequestVote", pRoot); @@ -929,44 +942,46 @@ SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { } cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - cJSON_AddNumberToObject(pRoot, "vote_granted", pMsg->voteGranted); + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "vote_granted", pMsg->voteGranted); + } cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncRequestVoteReply", pRoot); @@ -1072,62 +1087,64 @@ SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg) { } cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); - cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); - cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->commitIndex); - cJSON_AddStringToObject(pRoot, "commit_index", u64buf); - - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - char* s; - s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - free(s); - s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); + cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commit_index", u64buf); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + free(s); + s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + free(s); + } cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncAppendEntries", pRoot); @@ -1231,47 +1248,49 @@ SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg } cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "success", pMsg->success); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); + cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - cJSON_AddNumberToObject(pRoot, "success", pMsg->success); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); - cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncAppendEntriesReply", pRoot); diff --git a/source/libs/sync/src/syncOnMessage.c b/source/libs/sync/src/syncOnMessage.c index 19a97ee1566d32ed0fc8786c4d75542c2c435f30..ce8bed9cd39c44df9b090ae931cba063d1dda53c 100644 --- a/source/libs/sync/src/syncOnMessage.c +++ b/source/libs/sync/src/syncOnMessage.c @@ -14,3 +14,43 @@ */ #include "syncOnMessage.h" + +// TLA+ Spec +// Receive(m) == +// LET i == m.mdest +// j == m.msource +// IN \* Any RPC with a newer term causes the recipient to advance +// \* its term first. Responses with stale terms are ignored. +// \/ UpdateTerm(i, j, m) +// \/ /\ m.mtype = RequestVoteRequest +// /\ HandleRequestVoteRequest(i, j, m) +// \/ /\ m.mtype = RequestVoteResponse +// /\ \/ DropStaleResponse(i, j, m) +// \/ HandleRequestVoteResponse(i, j, m) +// \/ /\ m.mtype = AppendEntriesRequest +// /\ HandleAppendEntriesRequest(i, j, m) +// \/ /\ m.mtype = AppendEntriesResponse +// /\ \/ DropStaleResponse(i, j, m) +// \/ HandleAppendEntriesResponse(i, j, m) + +// DuplicateMessage(m) == +// /\ Send(m) +// /\ UNCHANGED <> + +// DropMessage(m) == +// /\ Discard(m) +// /\ UNCHANGED <> + +// Next == /\ \/ \E i \in Server : Restart(i) +// \/ \E i \in Server : Timeout(i) +// \/ \E i,j \in Server : RequestVote(i, j) +// \/ \E i \in Server : BecomeLeader(i) +// \/ \E i \in Server, v \in Value : ClientRequest(i, v) +// \/ \E i \in Server : AdvanceCommitIndex(i) +// \/ \E i,j \in Server : AppendEntries(i, j) +// \/ \E m \in DOMAIN messages : Receive(m) +// \/ \E m \in DOMAIN messages : DuplicateMessage(m) +// \/ \E m \in DOMAIN messages : DropMessage(m) +// \* History variable that tracks every log ever: +// /\ allLogs' = allLogs \cup {log[i] : i \in Server} +// \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index f29b3022d8e0b15a77d9a811b6a8db19288c697c..e4c5882e9896d423709e2fcc4c8de6245ad908b4 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -26,6 +26,7 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { return pEntry; } +// step 4. SyncClientRequest => SSyncRaftEntry, add term, index SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); assert(pEntry != NULL); @@ -48,6 +49,7 @@ void syncEntryDestory(SSyncRaftEntry* pEntry) { } } +// step 5. SSyncRaftEntry => bin, to raft log char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { char* buf = malloc(pEntry->bytes); assert(buf != NULL); @@ -58,6 +60,7 @@ char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { return buf; } +// step 6. bin => SSyncRaftEntry, from raft log SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SSyncRaftEntry* pEntry = malloc(bytes); @@ -68,29 +71,31 @@ SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { } cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { - char u64buf[128]; - + char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); - cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); - snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->seqNum); - cJSON_AddStringToObject(pRoot, "seqNum", u64buf); - cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); - snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index); - cJSON_AddStringToObject(pRoot, "index", u64buf); - cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); - - char* s; - s = syncUtilprintBin((char*)(pEntry->data), pEntry->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - free(s); - - s = syncUtilprintBin2((char*)(pEntry->data), pEntry->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + + if (pEntry != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); + cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); + snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->seqNum); + cJSON_AddStringToObject(pRoot, "seqNum", u64buf); + cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); + snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index); + cJSON_AddStringToObject(pRoot, "index", u64buf); + cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); + + char* s; + s = syncUtilprintBin((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + free(s); + + s = syncUtilprintBin2((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + free(s); + } cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot); @@ -104,29 +109,38 @@ char* syncEntry2Str(const SSyncRaftEntry* pEntry) { return serialized; } +// step 7. SSyncRaftEntry => original SRpcMsg, commit to user, delete seqNum, isWeak, term, index +void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pEntry->originalRpcType; + pRpcMsg->contLen = pEntry->dataLen; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen); +} + // for debug ---------------------- void syncEntryPrint(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); - printf("syncEntryPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); free(serialized); } void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); - printf("syncEntryPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); free(serialized); } void syncEntryLog(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); - sTrace("syncEntryLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); free(serialized); } void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); - sTrace("syncEntryLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); free(serialized); -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 6ebeba1991c3df5a70ee2349bb3482d2b318d776..620e923629863a844d3b1382d43e341426e10cc7 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -34,6 +34,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->getLastTerm = logStoreLastTerm; pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex; + return pLogStore; // to avoid compiler error } void logStoreDestory(SSyncLogStore* pLogStore) { @@ -58,20 +59,24 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { walFsync(pWal, true); free(serialized); + return code; // to avoid compiler error } SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - SSyncRaftEntry* pEntry; + SSyncRaftEntry* pEntry = NULL; - SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - walReadWithHandle(pWalHandle, index); - pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len); - assert(pEntry != NULL); + if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { + SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + walReadWithHandle(pWalHandle, index); + pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len); + assert(pEntry != NULL); + + // need to hold, do not new every time!! + walCloseReadHandle(pWalHandle); + } - // need to hold, do not new every time!! - walCloseReadHandle(pWalHandle); return pEntry; } @@ -79,6 +84,7 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; walRollback(pWal, fromIndex); + return 0; // to avoid compiler error } SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore) { @@ -102,6 +108,7 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; walCommit(pWal, index); + return 0; // to avoid compiler error } SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore) { @@ -122,26 +129,28 @@ SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore) { } cJSON* logStore2Json(SSyncLogStore* pLogStore) { - char u64buf[128]; - + char u64buf[128]; SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; cJSON* pRoot = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); - - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "pEntries", pEntries); - SyncIndex lastIndex = logStoreLastIndex(pLogStore); - for (SyncIndex i = 0; i <= lastIndex; ++i) { - SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - syncEntryDestory(pEntry); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "pEntries", pEntries); + SyncIndex lastIndex = logStoreLastIndex(pLogStore); + for (SyncIndex i = 0; i <= lastIndex; ++i) { + SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } } cJSON* pJson = cJSON_CreateObject(); @@ -181,4 +190,4 @@ void logStoreLog2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); sTrace("logStorePrint | len:%lu | %s | %s", strlen(serialized), s, serialized); free(serialized); -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 5ad618b9c097e7c088133314be3067c5f8d09819..3f6db129ce46de97b14da2b240b56878826f5e0e 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -97,16 +97,32 @@ int32_t raftStorePersist(SRaftStore *pRaftStore) { return 0; } -static bool raftStoreFileExist(char *path) { return taosStatFile(path, NULL, NULL) >= 0; } +static bool raftStoreFileExist(char *path) { + bool b = taosStatFile(path, NULL, NULL) >= 0; + return b; +} int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { assert(pRaftStore != NULL); cJSON *pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "current_term", pRaftStore->currentTerm); - cJSON_AddNumberToObject(pRoot, "vote_for_addr", pRaftStore->voteFor.addr); + + char u64Buf[128]; + snprintf(u64Buf, sizeof(u64Buf), "%lu", pRaftStore->currentTerm); + cJSON_AddStringToObject(pRoot, "current_term", u64Buf); + + snprintf(u64Buf, sizeof(u64Buf), "%lu", pRaftStore->voteFor.addr); + cJSON_AddStringToObject(pRoot, "vote_for_addr", u64Buf); + cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); + uint64_t u64 = pRaftStore->voteFor.addr; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pRoot, "addr_host", host); + cJSON_AddNumberToObject(pRoot, "addr_port", port); + char *serialized = cJSON_Print(pRoot); int len2 = strlen(serialized); assert(len2 < len); @@ -125,10 +141,12 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { cJSON *pRoot = cJSON_Parse(buf); cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); - pRaftStore->currentTerm = pCurrentTerm->valueint; + assert(cJSON_IsString(pCurrentTerm)); + sscanf(pCurrentTerm->valuestring, "%lu", &(pRaftStore->currentTerm)); cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); - pRaftStore->voteFor.addr = pVoteForAddr->valueint; + assert(cJSON_IsString(pVoteForAddr)); + sscanf(pVoteForAddr->valuestring, "%lu", &(pRaftStore->voteFor.addr)); cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); pRaftStore->voteFor.vgId = pVoteForVgid->valueint; @@ -139,11 +157,10 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { bool raftStoreHasVoted(SRaftStore *pRaftStore) { bool b = syncUtilEmptyId(&(pRaftStore->voteFor)); - return b; + return (!b); } void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId) { - assert(!raftStoreHasVoted(pRaftStore)); assert(!syncUtilEmptyId(pRaftId)); pRaftStore->voteFor = *pRaftId; raftStorePersist(pRaftStore); @@ -164,30 +181,68 @@ void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term) { raftStorePersist(pRaftStore); } +int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson) { return 0; } + +cJSON *raftStore2Json(SRaftStore *pRaftStore) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pRaftStore != NULL) { + snprintf(u64buf, sizeof(u64buf), "%lu", pRaftStore->currentTerm); + cJSON_AddStringToObject(pRoot, "currentTerm", u64buf); + + cJSON *pVoteFor = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pRaftStore->voteFor.addr); + cJSON_AddStringToObject(pVoteFor, "addr", u64buf); + { + uint64_t u64 = pRaftStore->voteFor.addr; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pVoteFor, "addr_host", host); + cJSON_AddNumberToObject(pVoteFor, "addr_port", port); + } + cJSON_AddNumberToObject(pVoteFor, "vgId", pRaftStore->voteFor.vgId); + cJSON_AddItemToObject(pRoot, "voteFor", pVoteFor); + + int hasVoted = raftStoreHasVoted(pRaftStore); + cJSON_AddNumberToObject(pRoot, "hasVoted", hasVoted); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftStore", pRoot); + return pJson; +} + +char *raftStore2Str(SRaftStore *pRaftStore) { + cJSON *pJson = raftStore2Json(pRaftStore); + char * serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + // for debug ------------------- void raftStorePrint(SRaftStore *pObj) { - char serialized[RAFT_STORE_BLOCK_SIZE]; - raftStoreSerialize(pObj, serialized, sizeof(serialized)); + char *serialized = raftStore2Str(pObj); printf("raftStorePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); + free(serialized); } void raftStorePrint2(char *s, SRaftStore *pObj) { - char serialized[RAFT_STORE_BLOCK_SIZE]; - raftStoreSerialize(pObj, serialized, sizeof(serialized)); + char *serialized = raftStore2Str(pObj); printf("raftStorePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); + free(serialized); } void raftStoreLog(SRaftStore *pObj) { - char serialized[RAFT_STORE_BLOCK_SIZE]; - raftStoreSerialize(pObj, serialized, sizeof(serialized)); + char *serialized = raftStore2Str(pObj); sTrace("raftStoreLog | len:%lu | %s", strlen(serialized), serialized); - fflush(NULL); + free(serialized); } void raftStoreLog2(char *s, SRaftStore *pObj) { - char serialized[RAFT_STORE_BLOCK_SIZE]; - raftStoreSerialize(pObj, serialized, sizeof(serialized)); + char *serialized = raftStore2Str(pObj); sTrace("raftStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - fflush(NULL); + free(serialized); } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index b935943a1dd7dab3c69d2ae4c923504905ad2b14..0bb701fc09a112234622a3f32830ba12e47445b7 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -18,6 +18,7 @@ #include "syncMessage.h" #include "syncRaftEntry.h" #include "syncRaftLog.h" +#include "syncRaftStore.h" #include "syncUtil.h" // TLA+ Spec @@ -50,33 +51,57 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { - SRaftId* pDestId = &(pSyncNode->peersId[i]); - SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + SRaftId* pDestId = &(pSyncNode->peersId[i]); + // set prevLogIndex + SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); SyncIndex preLogIndex = nextIndex - 1; + // set preLogTerm SyncTerm preLogTerm = 0; if (preLogIndex >= SYNC_INDEX_BEGIN) { SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preLogIndex); + assert(pPreEntry != NULL); + preLogTerm = pPreEntry->term; + syncEntryDestory(pPreEntry); } - SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); - assert(nextIndex == lastIndex); + // batch optimized + // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); + + SyncAppendEntries* pMsg = NULL; + SSyncRaftEntry* pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex); + if (pEntry != NULL) { + pMsg = syncAppendEntriesBuild(pEntry->bytes); + assert(pMsg != NULL); - SSyncRaftEntry* pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex); - assert(pEntry != NULL); + // add pEntry into msg + uint32_t len; + char* serialized = syncEntrySerialize(pEntry, &len); + assert(len == pEntry->bytes); + memcpy(pMsg->data, serialized, len); + + free(serialized); + syncEntryDestory(pEntry); + + } else { + // maybe overflow, send empty record + pMsg = syncAppendEntriesBuild(0); + assert(pMsg != NULL); + } - SyncAppendEntries* pMsg = syncAppendEntriesBuild(pEntry->bytes); + assert(pMsg != NULL); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = *pDestId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; pMsg->prevLogIndex = preLogIndex; pMsg->prevLogTerm = preLogTerm; pMsg->commitIndex = pSyncNode->commitIndex; - pMsg->dataLen = pEntry->bytes; - // add pEntry into msg + // send AppendEntries syncNodeAppendEntries(pSyncNode, pDestId, pMsg); + syncAppendEntriesDestroy(pMsg); } return ret; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index be4f40aaadd1aaa09bf3109341d59b988e63e646..e23748a81e7e8f8622c31d9bf98b381727a44c39 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -43,7 +43,10 @@ // int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; - syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteLog2(logBuf, pMsg); if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); @@ -56,6 +59,8 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); if (grant) { + // maybe has already voted for pMsg->srcId + // vote again, no harm raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); } diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 7cdeace1667577eeb781e1b0908e80405c1a2ded..1531f701ff7605543c6c5abaabcbb1d88d894528 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -38,13 +38,18 @@ // int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; - syncRequestVoteReplyLog2("==syncNodeOnRequestVoteReplyCb==", pMsg); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteReplyLog2(logBuf, pMsg); if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("DropStaleResponse, receive term:%lu, current term:%lu", pMsg->term, ths->pRaftStore->currentTerm); + sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, + ths->pRaftStore->currentTerm); return ret; } + assert(!(pMsg->term > ths->pRaftStore->currentTerm)); // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { // syncNodeUpdateTerm(ths, pMsg->term); @@ -52,17 +57,29 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) assert(pMsg->term == ths->pRaftStore->currentTerm); + // This tallies votes even when the current state is not Candidate, + // but they won't be looked at, so it doesn't matter. if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { votesRespondAdd(ths->pVotesRespond, pMsg); if (pMsg->voteGranted) { + // add vote voteGrantedVote(ths->pVotesGranted, pMsg); + + // maybe to leader if (voteGrantedMajority(ths->pVotesGranted)) { - if (ths->pVotesGranted->toLeader) { + if (!ths->pVotesGranted->toLeader) { syncNodeCandidate2Leader(ths); + + // prevent to leader again! ths->pVotesGranted->toLeader = true; } } + } else { + ; + // do nothing + // UNCHANGED <> } } + return ret; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 3a48b0cbb35bfcb56d3a45373360f2b8e59ec69c..0d3a3c3cc5230d15dccd4be6fb8b399fed114bff 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -24,7 +24,8 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) { if (pMsg->timeoutType == SYNC_TIMEOUT_PING) { if (atomic_load_64(&ths->pingTimerLogicClockUser) <= pMsg->logicClock) { ++(ths->pingTimerCounter); - syncNodePingAll(ths); + // syncNodePingAll(ths); + syncNodePingPeers(ths); } } else if (pMsg->timeoutType == SYNC_TIMEOUT_ELECTION) { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index ba8a76c1901750ae9687e22e2bf12165a8fb9238..9486405331bd3d5352eb8b1ce73cbff950b3feaa 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -14,15 +14,12 @@ */ #include "syncUtil.h" -#include -#include -#include #include "syncEnv.h" // ---- encode / decode uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { uint64_t u64; - uint32_t hostU32 = (uint32_t)inet_addr(host); + uint32_t hostU32 = (uint32_t)taosInetAddr(host); // assert(hostU32 != (uint32_t)-1); u64 = (((uint64_t)hostU32) << 32) | (((uint32_t)port) << 16); return u64; @@ -33,7 +30,7 @@ void syncUtilU642Addr(uint64_t u64, char* host, size_t len, uint16_t* port) { struct in_addr addr; addr.s_addr = hostU32; - snprintf(host, len, "%s", inet_ntoa(addr)); + snprintf(host, len, "%s", taosInetNtoa(addr)); *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); } @@ -119,7 +116,7 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) { char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", p->addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", p->addr); cJSON_AddStringToObject(pRoot, "addr", u64buf); char host[128]; uint16_t port; @@ -196,4 +193,4 @@ SyncIndex syncUtilMinIndex(SyncIndex a, SyncIndex b) { SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b) { SyncIndex r = a > b ? a : b; return r; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 5c8e70979c14b0fe8eff06aa2fffa33c3c34581d..69a4c48f39f9b532b106a3ba4b55871361b19a38 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -82,30 +82,32 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { char u64buf[128]; cJSON *pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int i = 0; i < pVotesGranted->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); - } - int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum); - for (int i = 0; i < pVotesGranted->replicaNum; ++i) { - arr[i] = pVotesGranted->isGranted[i]; + if (pVotesGranted != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int i = 0; i < pVotesGranted->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); + } + int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum); + for (int i = 0; i < pVotesGranted->replicaNum; ++i) { + arr[i] = pVotesGranted->isGranted[i]; + } + cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); + free(arr); + cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); + + cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); + snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); + cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + bool majority = voteGrantedMajority(pVotesGranted); + cJSON_AddNumberToObject(pRoot, "majority", majority); } - cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); - free(arr); - cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); - - cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); - snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); - cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - bool majority = voteGrantedMajority(pVotesGranted); - cJSON_AddNumberToObject(pRoot, "majority", majority); cJSON *pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); @@ -203,29 +205,31 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { char u64buf[128]; cJSON *pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int i = 0; i < pVotesRespond->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); - } - int respondNum = 0; - int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum); - for (int i = 0; i < pVotesRespond->replicaNum; ++i) { - arr[i] = pVotesRespond->isRespond[i]; - if (pVotesRespond->isRespond[i]) { - respondNum++; + if (pVotesRespond != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int i = 0; i < pVotesRespond->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); } + int respondNum = 0; + int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum); + for (int i = 0; i < pVotesRespond->replicaNum; ++i) { + arr[i] = pVotesRespond->isRespond[i]; + if (pVotesRespond->isRespond[i]) { + respondNum++; + } + } + cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); + free(arr); + cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); + cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); + + snprintf(u64buf, sizeof(u64buf), "%lu", pVotesRespond->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); } - cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); - free(arr); - cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); - cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); - - snprintf(u64buf, sizeof(u64buf), "%lu", pVotesRespond->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); cJSON *pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); @@ -264,4 +268,4 @@ void votesRespondLog2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); sTrace("votesRespondLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); free(serialized); -} \ No newline at end of file +} diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 6ade78936d2e67b426acb0a8981c7a4283076456..0c7608f902621ba8b233f67a6844fdf953b360fa 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -25,6 +25,12 @@ add_executable(syncTimeoutTest "") add_executable(syncPingTest "") add_executable(syncPingReplyTest "") add_executable(syncRpcMsgTest "") +add_executable(syncPingTimerTest2 "") +add_executable(syncPingSelfTest "") +add_executable(syncElectTest "") +add_executable(syncEncodeTest "") +add_executable(syncWriteTest "") +add_executable(syncReplicateTest "") target_sources(syncTest @@ -135,6 +141,30 @@ target_sources(syncRpcMsgTest PRIVATE "syncRpcMsgTest.cpp" ) +target_sources(syncPingTimerTest2 + PRIVATE + "syncPingTimerTest2.cpp" +) +target_sources(syncPingSelfTest + PRIVATE + "syncPingSelfTest.cpp" +) +target_sources(syncElectTest + PRIVATE + "syncElectTest.cpp" +) +target_sources(syncEncodeTest + PRIVATE + "syncEncodeTest.cpp" +) +target_sources(syncWriteTest + PRIVATE + "syncWriteTest.cpp" +) +target_sources(syncReplicateTest + PRIVATE + "syncReplicateTest.cpp" +) target_include_directories(syncTest @@ -272,6 +302,41 @@ target_include_directories(syncRpcMsgTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncPingTimerTest2 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncPingSelfTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncElectTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncElectTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncEncodeTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncWriteTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncReplicateTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -382,6 +447,30 @@ target_link_libraries(syncRpcMsgTest sync gtest_main ) +target_link_libraries(syncPingTimerTest2 + sync + gtest_main +) +target_link_libraries(syncPingSelfTest + sync + gtest_main +) +target_link_libraries(syncElectTest + sync + gtest_main +) +target_link_libraries(syncEncodeTest + sync + gtest_main +) +target_link_libraries(syncWriteTest + sync + gtest_main +) +target_link_libraries(syncReplicateTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0e0b57a0257001c717e3a0f23de5fbf2cac31cb7 --- /dev/null +++ b/source/libs/sync/test/syncElectTest.cpp @@ -0,0 +1,125 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* gSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./elect_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + gSyncNode = syncInitTest(); + assert(gSyncNode != NULL); + syncNodePrint2((char*)"", gSyncNode); + + initRaftId(gSyncNode); + + //--------------------------- + while (1) { + sTrace("elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock, + gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + } + + return 0; +} diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e554858072bf6a3f71c8ec1c185ead2274a0b852 --- /dev/null +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -0,0 +1,204 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM * pFsm; +SWal * pWal; +SSyncNode *pSyncNode; + +SSyncNode *syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen("./wal_test", &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode *syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +SRpcMsg *step0() { + SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 32; + pMsg->pCont = malloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +SRpcMsg *step2(const SyncClientRequest *pMsg) { + SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + syncClientRequest2RpcMsg(pMsg, pRetMsg); + return pRetMsg; +} + +SyncClientRequest *step3(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestFromRpcMsg2(pMsg); + return pRetMsg; +} + +SSyncRaftEntry *step4(const SyncClientRequest *pMsg) { + SSyncRaftEntry *pRetMsg = syncEntryBuild2((SyncClientRequest *)pMsg, 100, 0); + return pRetMsg; +} + +char *step5(const SSyncRaftEntry *pMsg, uint32_t *len) { + char *pRetMsg = syncEntrySerialize(pMsg, len); + return pRetMsg; +} + +SSyncRaftEntry *step6(const char *pMsg, uint32_t len) { + SSyncRaftEntry *pRetMsg = syncEntryDeserialize(pMsg, len); + return pRetMsg; +} + +SRpcMsg *step7(const SSyncRaftEntry *pMsg) { + SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + syncEntry2OriginalRpc(pMsg, pRetMsg); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + taosRemoveDir("./wal_test"); + + // step0 + SRpcMsg *pMsg0 = step0(); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + // step1 + SyncClientRequest *pMsg1 = step1(pMsg0); + syncClientRequestPrint2((char *)"==step1==", pMsg1); + + // step2 + SRpcMsg *pMsg2 = step2(pMsg1); + syncRpcMsgPrint2((char *)"==step2==", pMsg2); + + // step3 + SyncClientRequest *pMsg3 = step3(pMsg2); + syncClientRequestPrint2((char *)"==step3==", pMsg3); + + // step4 + SSyncRaftEntry *pMsg4 = step4(pMsg3); + syncEntryPrint2((char *)"==step4==", pMsg4); + + // log, relog + SSyncNode *pSyncNode = syncNodeInit(); + assert(pSyncNode != NULL); + SSyncRaftEntry *pEntry = pMsg4; + pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry); + SSyncRaftEntry *pEntry2 = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, pEntry->index); + syncEntryPrint2((char *)"==pEntry2==", pEntry2); + + // step5 + uint32_t len; + char * pMsg5 = step5(pMsg4, &len); + char * s = syncUtilprintBin(pMsg5, len); + printf("==step5== [%s] \n", s); + free(s); + + // step6 + SSyncRaftEntry *pMsg6 = step6(pMsg5, len); + syncEntryPrint2((char *)"==step6==", pMsg6); + + // step7 + SRpcMsg *pMsg7 = step7(pMsg6); + syncRpcMsgPrint2((char *)"==step7==", pMsg7); + + return 0; +} diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index 7898fda8c068c7b83aa87c0a873e55c5974d8b07..a3e5f41c85765e771cc8ab36efd663117f4cecab 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -47,12 +47,11 @@ SSyncNode* syncNodeInit() { gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 1b05f76fa2971357c923c3c43ff66a9f1ebea759..c1cb66f382574f042bf910308a5da5e2d1a1b0b1 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -81,7 +81,7 @@ SSyncNode* syncNodeInit() { SSyncNode* syncInitTest() { return syncNodeInit(); } void logStoreTest() { - logStorePrint2((char*)"logStoreTest2", pSyncNode->pLogStore); + logStorePrint2((char*)"logStoreTest", pSyncNode->pLogStore); assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_INVALID); @@ -105,10 +105,10 @@ void logStoreTest() { assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_BEGIN); } } - logStorePrint(pSyncNode->pLogStore); + logStorePrint2((char*)"after appendEntry", pSyncNode->pLogStore); pSyncNode->pLogStore->truncate(pSyncNode->pLogStore, 3); - logStorePrint(pSyncNode->pLogStore); + logStorePrint2((char*)"after truncate 3", pSyncNode->pLogStore); } void initRaftId(SSyncNode* pSyncNode) { diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..05e4d99cb0cb44388f82bf154ff28769d5766162 --- /dev/null +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -0,0 +1,102 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + syncNodePrint2((char*)"", pSyncNode); + + initRaftId(pSyncNode); + + //--------------------------- + + while (1) { + syncNodePingSelf(pSyncNode); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index e69878632f809d0eb9014c69fd4576076babf385..20d4a9ce589bad5b399f727729a9fe7f21c0d94f 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -47,6 +47,7 @@ SSyncNode* syncNodeInit() { gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2a041f3f5ddf4a530127699a0e64645bfd40b1f4 --- /dev/null +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -0,0 +1,106 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + syncNodePrint2((char*)"", pSyncNode); + + initRaftId(pSyncNode); + + //--------------------------- + + sTrace("syncNodeStartPingTimer ..."); + ret = syncNodeStartPingTimer(pSyncNode); + assert(ret == 0); + + while (1) { + sTrace("while 1 sleep ..."); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 0c1c9b881ed0ecc7978d607d3e679f7f3f35ebc9..688802625ac42203b7f52a459b7440ff3324f8e8 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -3,6 +3,7 @@ #include #include "syncIO.h" #include "syncInt.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -13,6 +14,21 @@ void logTest() { sFatal("--- sync log test: fatal"); } +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 5; +int32_t myIndex = 0; +SRaftId ids[TSDB_MAX_REPLICA]; + +void initRaftId() { + for (int i = 0; i < replicaNum; ++i) { + ids[i].addr = syncUtilAddr2U64("127.0.0.1", ports[i]); + ids[i].vgId = 1234; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + int main() { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; @@ -20,23 +36,35 @@ int main() { logTest(); - SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); + initRaftId(); + + SRaftStore* pRaftStore = raftStoreOpen("./test_raft_store.json"); assert(pRaftStore != NULL); - raftStorePrint(pRaftStore); - -#if 0 - pRaftStore->currentTerm = 100; - pRaftStore->voteFor.addr = 200; - pRaftStore->voteFor.vgId = 300; - raftStorePersist(pRaftStore); - raftStorePrint(pRaftStore); -#endif - - ++(pRaftStore->currentTerm); - ++(pRaftStore->voteFor.addr); - ++(pRaftStore->voteFor.vgId); - raftStorePersist(pRaftStore); - raftStorePrint(pRaftStore); + raftStorePrint2((char*)"==raftStoreOpen==", pRaftStore); + + raftStoreSetTerm(pRaftStore, 100); + raftStorePrint2((char*)"==raftStoreSetTerm==", pRaftStore); + + raftStoreVote(pRaftStore, &ids[0]); + raftStorePrint2((char*)"==raftStoreVote==", pRaftStore); + + raftStoreClearVote(pRaftStore); + raftStorePrint2((char*)"==raftStoreClearVote==", pRaftStore); + + raftStoreVote(pRaftStore, &ids[1]); + raftStorePrint2((char*)"==raftStoreVote==", pRaftStore); + + raftStoreNextTerm(pRaftStore); + raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + + raftStoreNextTerm(pRaftStore); + raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + + raftStoreNextTerm(pRaftStore); + raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + + raftStoreNextTerm(pRaftStore); + raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); return 0; } diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6c4fab24252656a3f8079465f1b07d192420a537 --- /dev/null +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -0,0 +1,188 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM * pFsm; +SWal * pWal; +SSyncNode *gSyncNode; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { + printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); + syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { + printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); + syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { + printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); + syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf); +} + +void initFsm() { + pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +SSyncNode *syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode *pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode *syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = malloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + taosRemoveDir("./wal_test"); + + initFsm(); + + gSyncNode = syncInitTest(); + assert(gSyncNode != NULL); + syncNodePrint2((char *)"", gSyncNode); + + initRaftId(gSyncNode); + + for (int i = 0; i < 30; ++i) { + // step0 + SRpcMsg *pMsg0 = step0(i); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + // step1 + SyncClientRequest *pMsg1 = step1(pMsg0); + syncClientRequestPrint2((char *)"==step1==", pMsg1); + + SyncClientRequest *pSyncClientRequest = pMsg1; + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + gSyncNode->FpEqMsg(gSyncNode->queue, &rpcMsg); + + taosMsleep(1000); + sTrace("replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock, + gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + } + + while (1) { + sTrace("replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock, + gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2598abbddd86ac6813afd287246202d5439cb9c8 --- /dev/null +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -0,0 +1,180 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM * pFsm; +SWal * pWal; +SSyncNode *gSyncNode; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { + printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); + syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { + printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); + syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { + printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); + syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf); +} + +void initFsm() { + pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +SSyncNode *syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./write_test"); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen("./write_test_wal", &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode *pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode *syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +SRpcMsg *step0() { + SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 32; + pMsg->pCont = malloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + taosRemoveDir("./wal_test"); + + initFsm(); + + gSyncNode = syncInitTest(); + assert(gSyncNode != NULL); + syncNodePrint2((char *)"", gSyncNode); + + initRaftId(gSyncNode); + + // step0 + SRpcMsg *pMsg0 = step0(); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + // step1 + SyncClientRequest *pMsg1 = step1(pMsg0); + syncClientRequestPrint2((char *)"==step1==", pMsg1); + + for (int i = 0; i < 10; ++i) { + SyncClientRequest *pSyncClientRequest = pMsg1; + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + gSyncNode->FpEqMsg(gSyncNode->queue, &rpcMsg); + + taosMsleep(1000); + } + + while (1) { + sTrace("while 1 sleep"); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 3cb5a655728cc77e477352bcd2425fce14c434df..978649499aa094c30bdcd6cd62ca8a00c1e86df1 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,10 +1,17 @@ - -set(TDB_SUBDIRS "db") -foreach(TDB_SUBDIR ${TDB_SUBDIRS}) - aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC) -endforeach() - -add_library(tdb STATIC ${TDB_SRC}) +# tdb +add_library(tdb "") +target_sources(tdb + PRIVATE + "src/db/tdbPCache.c" + "src/db/tdbPager.c" + "src/db/tdbUtil.c" + "src/db/tdbBtree.c" + "src/db/tdbDb.c" + "src/db/tdbEnv.c" + # "src/db/tdbPage.c" + "src/page/tdbPage.c" + "src/page/tdbPageL.c" +) target_include_directories( tdb @@ -17,6 +24,7 @@ target_link_libraries( PUBLIC util ) +# for test if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index cc0d20ef3c22ac8450421472f6585201af47cd1b..467e40325e73505e91abf45046e03adff52561ec 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,44 +22,42 @@ extern "C" { #endif -typedef struct STDb TDB; -typedef struct STDbEnv TENV; -typedef struct STDbCurosr TDBC; +// typedef struct STDb TDB; +// typedef struct STDbEnv TENV; +// typedef struct STDbCurosr TDBC; -typedef int32_t pgsz_t; -typedef int32_t cachesz_t; +// typedef int32_t pgsz_t; +// typedef int32_t cachesz_t; -typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); +// typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); -// TEVN -int tdbEnvCreate(TENV **ppEnv, const char *rootDir); -int tdbEnvOpen(TENV *ppEnv); -int tdbEnvClose(TENV *pEnv); +// // TEVN +// int tdbEnvCreate(TENV **ppEnv, const char *rootDir); +// int tdbEnvOpen(TENV *ppEnv); +// int tdbEnvClose(TENV *pEnv); -int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize); -pgsz_t tdbEnvGetPageSize(TENV *pEnv); -cachesz_t tdbEnvGetCacheSize(TENV *pEnv); +// int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize); +// pgsz_t tdbEnvGetPageSize(TENV *pEnv); +// cachesz_t tdbEnvGetCacheSize(TENV *pEnv); -int tdbEnvBeginTxn(TENV *pEnv); -int tdbEnvCommit(TENV *pEnv); +// int tdbEnvBeginTxn(TENV *pEnv); +// int tdbEnvCommit(TENV *pEnv); -// TDB -int tdbCreate(TDB **ppDb); -int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv); -int tdbClose(TDB *pDb); -int tdbDrop(TDB *pDb); +// // TDB +// int tdbCreate(TDB **ppDb); +// int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv); +// int tdbClose(TDB *pDb); +// int tdbDrop(TDB *pDb); -int tdbSetKeyLen(TDB *pDb, int klen); -int tdbSetValLen(TDB *pDb, int vlen); -int tdbSetDup(TDB *pDb, int dup); -int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn); -int tdbGetKeyLen(TDB *pDb); -int tdbGetValLen(TDB *pDb); -int tdbGetDup(TDB *pDb); +// int tdbSetKeyLen(TDB *pDb, int klen); +// int tdbSetValLen(TDB *pDb, int vlen); +// int tdbSetDup(TDB *pDb, int dup); +// int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn); +// int tdbGetKeyLen(TDB *pDb); +// int tdbGetValLen(TDB *pDb); +// int tdbGetDup(TDB *pDb); -int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData); - -// TDBC +// int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/db/tdb.c b/source/libs/tdb/src/db/tdb.c deleted file mode 100644 index 65d4cf80cca5ed5386fc69fee8d9976b610293d3..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/db/tdb.c +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -struct STDb { - char dbname[TDB_MAX_DBNAME_LEN]; - SBTree * pBt; // current access method (may extend) - SPgFile * pPgFile; // backend page file this DB is using - TENV * pEnv; // TENV containing the DB - int klen; // key length if know - int vlen; // value length if know - bool dup; // dup mode - TdbKeyCmprFn cFn; // compare function -}; - -struct STDbCurosr { - SBtCursor *pBtCur; -}; - -static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); - -int tdbCreate(TDB **ppDb) { - TDB *pDb; - - // create the handle - pDb = (TDB *)calloc(1, sizeof(*pDb)); - if (pDb == NULL) { - return -1; - } - - pDb->klen = TDB_VARIANT_LEN; - pDb->vlen = TDB_VARIANT_LEN; - pDb->dup = false; - pDb->cFn = tdbDefaultKeyCmprFn; - - *ppDb = pDb; - return 0; -} - -static int tdbDestroy(TDB *pDb) { - if (pDb) { - free(pDb); - } - return 0; -} - -int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) { - int ret; - uint8_t fileid[TDB_FILE_ID_LEN]; - SPgFile * pPgFile; - SPgCache *pPgCache; - SBTree * pBt; - bool fileExist; - size_t dbNameLen; - pgno_t dbRootPgno; - char dbfname[128]; // TODO: make this as a macro or malloc on the heap - - ASSERT(pDb != NULL); - ASSERT(fname != NULL); - // TODO: Here we simply put an assert here. In the future, make `pEnv` - // can be set as NULL. - ASSERT(pEnv != NULL); - - // check the DB name - dbNameLen = 0; - if (dbname) { - dbNameLen = strlen(dbname); - if (dbNameLen >= TDB_MAX_DBNAME_LEN) { - return -1; - } - - memcpy(pDb->dbname, dbname, dbNameLen); - } - - pDb->dbname[dbNameLen] = '\0'; - - // get page file from the env, if not opened yet, open it - pPgFile = NULL; - snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname); - fileExist = taosCheckExistFile(fname); - if (fileExist) { - tdbGnrtFileID(dbfname, fileid, false); - pPgFile = tdbEnvGetPageFile(pEnv, fileid); - } - - if (pPgFile == NULL) { - ret = pgFileOpen(&pPgFile, dbfname, pEnv); - if (ret != 0) { - // TODO: handle error - return -1; - } - } - - // TODO: get the root page number from the master DB of the page file - // tdbGet(&dbRootPgno); - if (dbRootPgno == 0) { - // DB not exist, create one - ret = pgFileAllocatePage(pPgFile, &dbRootPgno); - if (ret != 0) { - // TODO: handle error - } - // tdbInsert(pPgFile->pMasterDB, dbname, strlen(dbname), &dbRootPgno, sizeof(dbRootPgno)); - } - - ASSERT(dbRootPgno > 1); - - // pDb->pBt->root = dbRootPgno; - - // register - pDb->pPgFile = pPgFile; - tdbEnvRgstDB(pEnv, pDb); - pDb->pEnv = pEnv; - - return 0; -} - -int tdbClose(TDB *pDb) { - if (pDb == NULL) return 0; - return tdbDestroy(pDb); -} - -int tdbDrop(TDB *pDb) { - // TODO - return 0; -} - -int tdbSetKeyLen(TDB *pDb, int klen) { - // TODO: check `klen` - pDb->klen = klen; - return 0; -} - -int tdbSetValLen(TDB *pDb, int vlen) { - // TODO: check `vlen` - pDb->vlen = vlen; - return 0; -} - -int tdbSetDup(TDB *pDb, int dup) { - if (dup) { - pDb->dup = true; - } else { - pDb->dup = false; - } - return 0; -} - -int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn) { - if (fn == NULL) { - return -1; - } else { - pDb->cFn = fn; - } - return 0; -} - -int tdbGetKeyLen(TDB *pDb) { return pDb->klen; } - -int tdbGetValLen(TDB *pDb) { return pDb->vlen; } - -int tdbGetDup(TDB *pDb) { - if (pDb->dup) { - return 1; - } else { - return 0; - } -} - -int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData) { - // TODO - return 0; -} - -static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2) { - int mlen; - int cret; - - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); - - mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; - cret = memcmp(pKey1, pKey2, mlen); - if (cret == 0) { - if (keyLen1 < keyLen2) { - cret = -1; - } else if (keyLen1 > keyLen2) { - cret = 1; - } else { - cret = 0; - } - } - return cret; -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 86e7980733f39baa393cc45d87cedf755593900e..c6e8c9dca9496f1e530c1caabeee3f35ef69c1ae 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -15,44 +15,99 @@ #include "tdbInt.h" -struct SBtCursor { - SBTree *pBtree; - pgno_t pgno; - SPage * pPage; // current page traversing +#define TDB_BTREE_ROOT 0x1 +#define TDB_BTREE_LEAF 0x2 + +#define TDB_BTREE_PAGE_IS_ROOT(flags) TDB_FLAG_HAS(flags, TDB_BTREE_ROOT) +#define TDB_BTREE_PAGE_IS_LEAF(flags) TDB_FLAG_HAS(flags, TDB_BTREE_LEAF) +#define TDB_BTREE_ASSERT_FLAG(flags) \ + ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ + TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0)) + +struct SBTree { + SPgno root; + int keyLen; + int valLen; + SPager *pPager; + FKeyComparator kcmpr; + u8 fanout; + int pageSize; + int maxLocal; + int minLocal; + int maxLeaf; + int minLeaf; + u8 *pTmp; }; +typedef struct __attribute__((__packed__)) { + SPgno rChild; +} SBtPageHdr; + +typedef struct { + u16 flags; + SBTree *pBt; +} SBtreeZeroPageArg; + typedef struct { - pgno_t pgno; - pgsz_t offset; -} SBtIdx; + int kLen; + u8 *pKey; + int vLen; + u8 *pVal; + SPgno pgno; + u8 *pTmpSpace; +} SCellDecoder; -// Btree page header definition -typedef struct __attribute__((__packed__)) { - uint8_t flag; // page flag - int32_t vlen; // value length of current page, TDB_VARIANT_LEN for variant length - uint16_t nPayloads; // number of total payloads - pgoff_t freeOff; // free payload offset - pgsz_t fragSize; // total fragment size - pgoff_t offPayload; // payload offset - pgno_t rChildPgno; // right most child page number -} SBtPgHdr; - -typedef int (*BtreeCmprFn)(const void *, const void *); - -#define BTREE_PAGE_HDR(pPage) NULL /* TODO */ -#define BTREE_PAGE_PAYLOAD_AT(pPage, idx) NULL /*TODO*/ -#define BTREE_PAGE_IS_LEAF(pPage) 0 /* TODO */ - -static int btreeCreate(SBTree **ppBt); -static int btreeDestroy(SBTree *pBt); -static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno); - -int btreeOpen(SBTree **ppBt, SPgFile *pPgFile) { +static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst); +static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); +static int tdbBtreeOpenImpl(SBTree *pBt); +static int tdbBtreeZeroPage(SPage *pPage, void *arg); +static int tdbBtreeInitPage(SPage *pPage, void *arg); +static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, + int *szCell); +static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); +static int tdbBtreeBalance(SBtCursor *pCur); + +int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; int ret; - ret = btreeCreate(&pBt); - if (ret != 0) { + *ppBt = NULL; + + pBt = (SBTree *)calloc(1, sizeof(*pBt)); + if (pBt == NULL) { + return -1; + } + + // pBt->keyLen + pBt->keyLen = keyLen; + // pBt->valLen + pBt->valLen = valLen; + // pBt->pPager + pBt->pPager = pPager; + // pBt->kcmpr + pBt->kcmpr = kcmpr ? kcmpr : tdbDefaultKeyCmprFn; + // pBt->fanout + if (keyLen == TDB_VARIANT_LEN) { + pBt->fanout = TDB_DEFAULT_FANOUT; + } else { + ASSERT(0); + // TODO: pBt->fanout = 0; + } + // pBt->pageSize + pBt->pageSize = tdbPagerGetPageSize(pPager); + // pBt->maxLocal + pBt->maxLocal = (pBt->pageSize - 14) / pBt->fanout; + // pBt->minLocal: Should not be allowed smaller than 15, which is [nPayload][nKey][nData] + pBt->minLocal = (pBt->pageSize - 14) / pBt->fanout / 2; + // pBt->maxLeaf + pBt->maxLeaf = pBt->pageSize - 14; + // pBt->minLeaf + pBt->minLeaf = pBt->minLocal; + + // TODO: pBt->root + ret = tdbBtreeOpenImpl(pBt); + if (ret < 0) { + free(pBt); return -1; } @@ -60,105 +115,828 @@ int btreeOpen(SBTree **ppBt, SPgFile *pPgFile) { return 0; } -int btreeClose(SBTree *pBt) { +int tdbBtreeClose(SBTree *pBt) { // TODO return 0; } -static int btreeCreate(SBTree **ppBt) { +int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { + pCur->pBt = pBt; + pCur->iPage = -1; + pCur->pPage = NULL; + pCur->idx = -1; + + return 0; +} + +int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { + int ret; + int idx; + SPager *pPager; + SCell *pCell; + int szCell; + int cret; SBTree *pBt; - pBt = (SBTree *)calloc(1, sizeof(*pBt)); - if (pBt == NULL) { + ret = tdbBtCursorMoveTo(pCur, pKey, kLen, &cret); + if (ret < 0) { + // TODO: handle error return -1; } + if (pCur->idx == -1) { + ASSERT(TDB_PAGE_NCELLS(pCur->pPage) == 0); + idx = 0; + } else { + if (cret > 0) { + idx = pCur->idx + 1; + } else if (cret < 0) { + idx = pCur->idx; + } else { + /* TODO */ + ASSERT(0); + } + } + + // TODO: refact code here + pBt = pCur->pBt; + if (!pBt->pTmp) { + pBt->pTmp = (u8 *)malloc(pBt->pageSize); + if (pBt->pTmp == NULL) { + return -1; + } + } + + pCell = pBt->pTmp; + + // Encode the cell + ret = tdbBtreeEncodeCell(pCur->pPage, pKey, kLen, pVal, vLen, pCell, &szCell); + if (ret < 0) { + return -1; + } + + // Insert the cell to the index + ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell); + if (ret < 0) { + return -1; + } + + // If page is overflow, balance the tree + if (pCur->pPage->nOverflow > 0) { + ret = tdbBtreeBalance(pCur); + if (ret < 0) { + return -1; + } + } + + return 0; +} + +static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { // TODO return 0; } -static int btreeDestroy(SBTree *pBt) { - if (pBt) { - free(pBt); +static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst) { + int ret; + SBTree *pBt; + SPager *pPager; + + pBt = pCur->pBt; + pPager = pBt->pPager; + + if (pCur->iPage < 0) { + ASSERT(pCur->iPage == -1); + ASSERT(pCur->idx == -1); + + // Move from the root + ret = tdbPagerFetchPage(pPager, pBt->root, &(pCur->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pCur->iPage = 0; + + if (TDB_PAGE_NCELLS(pCur->pPage) == 0) { + // Current page is empty + ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); + return 0; + } + + for (;;) { + int lidx, ridx, midx, c, nCells; + SCell *pCell; + SPage *pPage; + SCellDecoder cd = {0}; + + pPage = pCur->pPage; + nCells = TDB_PAGE_NCELLS(pPage); + lidx = 0; + ridx = nCells - 1; + + ASSERT(nCells > 0); + + for (;;) { + if (lidx > ridx) break; + + midx = (lidx + ridx) >> 1; + + pCell = TDB_PAGE_CELL_AT(pPage, midx); + ret = tdbBtreeDecodeCell(pPage, pCell, &cd); + if (ret < 0) { + // TODO: handle error + ASSERT(0); + return -1; + } + + // Compare the key values + c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + if (c < 0) { + /* input-key < cell-key */ + ridx = midx - 1; + } else if (c > 0) { + /* input-key > cell-key */ + lidx = midx + 1; + } else { + /* input-key == cell-key */ + break; + } + } + + // Move downward or break + u16 flags = TDB_PAGE_FLAGS(pPage); + u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + if (leaf) { + pCur->idx = midx; + *pCRst = c; + break; + } else { + if (c <= 0) { + pCur->idx = midx; + tdbBtCursorMoveToChild(pCur, cd.pgno); + } else { + if (midx == nCells - 1) { + /* Move to right-most child */ + pCur->idx = midx + 1; + tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + } else { + // TODO: reset cd as uninitialized + pCur->idx = midx + 1; + pCell = TDB_PAGE_CELL_AT(pPage, midx + 1); + tdbBtreeDecodeCell(pPage, pCell, &cd); + tdbBtCursorMoveToChild(pCur, cd.pgno); + } + } + } + } + + } else { + // TODO: Move the cursor from a some position instead of a clear state + ASSERT(0); } + return 0; } -int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt) { - // TODO +static int tdbBtCursorMoveToRoot(SBtCursor *pCur) { + SBTree *pBt; + SPager *pPager; + SPage *pPage; + int ret; + + pBt = pCur->pBt; + pPager = pBt->pPager; + + // pPage = tdbPagerGet(pPager, pBt->root, true); + // if (pPage == NULL) { + // // TODO: handle error + // } + + // ret = tdbInitBtPage(pPage, &pBtPage); + // if (ret < 0) { + // // TODO + // return 0; + // } + + // pCur->pPage = pBtPage; + // pCur->iPage = 0; + return 0; } -int btreeCursorClose(SBtCursor *pBtCur) { - // TODO - return 0; +static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; } -int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey) { - SPage * pPage; - SBtPgHdr * pBtPgHdr; - SPgFile * pPgFile; - pgno_t childPgno; - pgno_t rootPgno; - int nPayloads; - void * pPayload; - BtreeCmprFn cmpFn; +static int tdbBtreeOpenImpl(SBTree *pBt) { + // Try to get the root page of the an existing btree + + SPgno pgno; + SPage *pPage; + int ret; - // 1. Move the cursor to the root page - if (rootPgno == TDB_IVLD_PGNO) { - // No any data in this btree, just return not found (TODO) + { + // 1. TODO: Search the main DB to check if the DB exists + pgno = 0; + } + + if (pgno != 0) { + pBt->root = pgno; return 0; + } + + // Try to create a new database + SBtreeZeroPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; + ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg); + if (ret < 0) { + return -1; + } + + // TODO: Unref the page + + ASSERT(pgno != 0); + pBt->root = pgno; + + return 0; +} + +static int tdbBtreeInitPage(SPage *pPage, void *arg) { + SBTree *pBt; + u16 flags; + u8 isLeaf; + + pBt = (SBTree *)arg; + + flags = TDB_PAGE_FLAGS(pPage); + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + if (isLeaf) { + pPage->szAmHdr = 0; + } else { + pPage->szAmHdr = sizeof(SBtPageHdr); + } + pPage->pPageHdr = pPage->pData; + pPage->pAmHdr = pPage->pPageHdr + pPage->pPageMethods->szPageHdr; + pPage->pCellIdx = pPage->pAmHdr + pPage->szAmHdr; + pPage->pFreeStart = pPage->pCellIdx + pPage->pPageMethods->szOffset * TDB_PAGE_NCELLS(pPage); + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + + TDB_BTREE_ASSERT_FLAG(flags); + + // Init other fields + if (isLeaf) { + pPage->kLen = pBt->keyLen; + pPage->vLen = pBt->valLen; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; } else { - // Load the page from the file by the SPgFile handle - pPage = pgFileFetch(pPgFile, rootPgno); + pPage->kLen = pBt->keyLen; + pPage->vLen = sizeof(SPgno); + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + } + + // TODO: need to update the SPage.nFree + pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; + pPage->nOverflow = 0; + + return 0; +} + +static int tdbBtreeZeroPage(SPage *pPage, void *arg) { + u16 flags; + SBTree *pBt; + + flags = ((SBtreeZeroPageArg *)arg)->flags; + pBt = ((SBtreeZeroPageArg *)arg)->pBt; + + pPage->pPageHdr = pPage->pData; + + // Init the page header + TDB_PAGE_FLAGS_SET(pPage, flags); + TDB_PAGE_NCELLS_SET(pPage, 0); + TDB_PAGE_CCELLS_SET(pPage, pBt->pageSize - sizeof(SPageFtr)); + TDB_PAGE_FCELL_SET(pPage, 0); + TDB_PAGE_NFREE_SET(pPage, 0); + + tdbBtreeInitPage(pPage, (void *)pBt); + + return 0; +} + +#ifndef TDB_BTREE_BALANCE +typedef struct { + SBTree *pBt; + SPage *pParent; + int idx; + i8 nOld; + SPage *pOldPages[3]; + i8 nNewPages; + SPage *pNewPages[5]; +} SBtreeBalanceHelper; + +static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { + int nCells = TDB_PAGE_NCELLS(pFrom); + int cCells = TDB_PAGE_CCELLS(pFrom); + int fCell = TDB_PAGE_FCELL(pFrom); + int nFree = TDB_PAGE_NFREE(pFrom); + + pTo->pFreeStart = pTo->pCellIdx + nCells * pFrom->pPageMethods->szOffset; + memcpy(pTo->pCellIdx, pFrom->pCellIdx, nCells * pFrom->pPageMethods->szOffset); + pTo->pFreeEnd = (u8 *)pTo->pPageFtr - (u8 *)(pFrom->pPageFtr) + pFrom->pFreeEnd; + memcpy(pTo->pFreeEnd, pFrom->pFreeEnd, (u8 *)pFrom->pPageFtr - pFrom->pFreeEnd); + + TDB_PAGE_NCELLS_SET(pTo, nCells); + TDB_PAGE_CCELLS_SET(pTo, cCells); + TDB_PAGE_FCELL_SET(pTo, fCell); + TDB_PAGE_NFREE_SET(pTo, nFree); + + // TODO: update other fields + + return 0; +} + +static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { + SPager *pPager; + SPage *pChild; + SPgno pgnoChild; + int ret; + SBtreeZeroPageArg zArg; + + pPager = pRoot->pPager; + + // Allocate a new child page + zArg.flags = TDB_BTREE_LEAF; + zArg.pBt = pBt; + ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg); + if (ret < 0) { + return -1; + } + + // Copy the root page content to the child page + ret = tdbBtreeCopyPageContent(pRoot, pChild); + if (ret < 0) { + return -1; + } + + pChild->nOverflow = pRoot->nOverflow; + for (int i = 0; i < pChild->nOverflow; i++) { + pChild->apOvfl[i] = pRoot->apOvfl[i]; + pChild->aiOvfl[i] = pRoot->aiOvfl[i]; + } + + // Reinitialize the root page + zArg.flags = TDB_BTREE_ROOT; + zArg.pBt = pBt; + ret = tdbBtreeZeroPage(pRoot, &zArg); + if (ret < 0) { + return -1; + } + + ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + + *ppChild = pChild; + return 0; +} - pBtCur->pPage = pPage; +static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { + int nCells; + int i; + int idxStart; + int nChild; + int ret; + SPage *pParent; + SPgno pgno; + SCell *pCell; + SCellDecoder cd; + SBTree *pBt; + + pParent = pBlh->pParent; + nCells = TDB_PAGE_NCELLS(pParent); + nChild = nCells + 1; + pBt = pBlh->pBt; + + // TODO: ASSERT below needs to be removed + ASSERT(pParent->nOverflow == 0); + ASSERT(pBlh->idx <= nCells); + + if (nChild < 3) { + idxStart = 0; + pBlh->nOld = nChild; + } else { + if (pBlh->idx == 0) { + idxStart = 0; + } else if (pBlh->idx == nCells) { + idxStart = pBlh->idx - 2; + } else { + idxStart = pBlh->idx - 1; + } + pBlh->nOld = 3; } - // 2. Loop to search over the whole tree + i = pBlh->nOld - 1; + + if (idxStart + i == nCells) { + pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; + } else { + pCell = TDB_PAGE_CELL_AT(pParent, idxStart + i); + // TODO: no need to decode the payload part, and even the kLen, vLen part + // we only need the pgno part + ret = tdbBtreeDecodeCell(pParent, pCell, &cd); + if (ret < 0) { + ASSERT(0); + return -1; + } + pgno = cd.pgno; + } for (;;) { - int lidx, ridx, midx, cret; + ret = tdbPagerFetchPage(pBt->pPager, pgno, &(pBlh->pOldPages[i]), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Loop over + if ((i--) == 0) break; + + { + // TODO + // ASSERT(0); + } + } + + return 0; +} + +static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { +#if 0 + SPage *pPage; + int oidx; + int cidx; + int limit; + SCell *pCell; - pPage = pBtCur->pPage; - pBtPgHdr = BTREE_PAGE_HDR(pPage); - nPayloads = pBtPgHdr->nPayloads; + for (int i = 0; i < pBlh->nOld; i++) { + pPage = pBlh->pOldPages[i]; + oidx = 0; + cidx = 0; - // Binary search the page - lidx = 0; - ridx = nPayloads - 1; - midx = (lidx + ridx) >> 1; + if (oidx < pPage->nOverflow) { + limit = pPage->aiOvfl[oidx]; + } else { + limit = pPage->pPageHdr->nCells; + } + + // Loop to copy each cell pointer out for (;;) { - // get the payload ptr at midx - pPayload = BTREE_PAGE_PAYLOAD_AT(pPage, midx); + if (oidx >= pPage->nOverflow && cidx >= pPage->pPageHdr->nCells) break; - // the payload and the key - cret = cmpFn(pKey, pPayload); + if (cidx < limit) { + // Get local cells + pCell = TDB_PAGE_CELL_AT(pPage, cidx); + } else if (cidx == limit) { + // Get overflow cells + pCell = pPage->apOvfl[oidx++]; - if (cret < 0) { - /* TODO */ - } else if (cret > 0) { - /* TODO */ + if (oidx < pPage->nOverflow) { + limit = pPage->aiOvfl[oidx]; + } else { + limit = pPage->pPageHdr->nCells; + } } else { - /* TODO */ + ASSERT(0); } + } - if (lidx > ridx) break; - midx = (lidx + ridx) >> 1; + { + // TODO: Copy divider cells here } - if (BTREE_PAGE_IS_LEAF(pPage)) { - /* TODO */ + } + + /* TODO */ + +#endif + return 0; +} + +static int tdbBtreeBalanceStep3(SBtreeBalanceHelper *pBlh) { + // Figure out number of pages needed after balance + for (int i = 0; i < pBlh->nOld; i++) { + /* TODO */ + } + + return 0; +} + +static int tdbBtreeBalanceStep4(SBtreeBalanceHelper *pBlh) { + // TODO + return 0; +} + +static int tdbBtreeBalanceStep5(SBtreeBalanceHelper *pBlh) { + // TODO + return 0; +} + +static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { + // TODO + return 0; +} + +static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { + int ret; + SBtreeBalanceHelper blh; + + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); + + blh.pBt = pBt; + blh.pParent = pParent; + blh.idx = idx; + + // Step 1: find two sibling pages and get engough info about the old pages + ret = tdbBtreeBalanceStep1(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 2: Load all cells on the old page and the divider cells + ret = tdbBtreeBalanceStep2(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 3: Get the number of pages needed to hold all cells + ret = tdbBtreeBalanceStep3(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 4: Allocate enough new pages. Reuse old pages as much as possible + ret = tdbBtreeBalanceStep4(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 5: Insert new divider cells into pParent + ret = tdbBtreeBalanceStep5(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // Step 6: Update the sibling pages + ret = tdbBtreeBalanceStep6(&blh); + if (ret < 0) { + ASSERT(0); + return -1; + } + + { + // TODO: Reset states + } + + { + // TODO: Clear resources + } + + return 0; +} + +static int tdbBtreeBalance(SBtCursor *pCur) { + int iPage; + SPage *pParent; + SPage *pPage; + int ret; + u16 flags; + u8 leaf; + u8 root; + + // Main loop to balance the BTree + for (;;) { + iPage = pCur->iPage; + pPage = pCur->pPage; + flags = TDB_PAGE_FLAGS(pPage); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + root = TDB_BTREE_PAGE_IS_ROOT(flags); + + // TODO: Get the page free space if not get yet + // if (pPage->nFree < 0) { + // if (tdbBtreeComputeFreeSpace(pPage) < 0) { + // return -1; + // } + // } + + // when the page is not overflow and not too empty, the balance work + // is finished. Just break out the balance loop. + if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) { break; + } + + if (iPage == 0) { + // For the root page, only balance when the page is overfull, + // ignore the case of empty + if (pPage->nOverflow == 0) break; + + ret = tdbBtreeBalanceDeeper(pCur->pBt, pCur->pPage, &(pCur->pgStack[1])); + if (ret < 0) { + return -1; + } + + pCur->idx = 0; + pCur->idxStack[0] = 0; + pCur->pgStack[0] = pCur->pPage; + pCur->iPage = 1; + pCur->pPage = pCur->pgStack[1]; } else { - /* TODO */ - btreeCursorMoveToChild(pBtCur, childPgno); + // Generalized balance step + pParent = pCur->pgStack[iPage - 1]; + + ret = tdbBtreeBalanceNonRoot(pCur->pBt, pParent, pCur->idxStack[pCur->iPage - 1]); + if (ret < 0) { + return -1; + } + + pCur->iPage--; + pCur->pPage = pCur->pgStack[pCur->iPage]; } } return 0; } +#endif + +#ifndef TDB_BTREE_CELL // ========================================================= +static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, int kLen, const void *pVal, int vLen, + int *szPayload) { + int nPayload; + + ASSERT(pKey != NULL); + + if (pVal == NULL) { + vLen = 0; + } + + nPayload = kLen + vLen; + if (nPayload <= pPage->maxLocal) { + // General case without overflow + memcpy(pPayload, pKey, kLen); + if (pVal) { + memcpy(pPayload + kLen, pVal, vLen); + } + + *szPayload = nPayload; + return 0; + } + + { + // TODO: handle overflow case + ASSERT(0); + } -static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno) { - SPgFile *pPgFile; - // TODO return 0; -} \ No newline at end of file +} + +static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, + int *szCell) { + u16 flags; + u8 leaf; + int nHeader; + int nPayload; + int ret; + + ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen); + ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen); + + nPayload = 0; + nHeader = 0; + flags = TDB_PAGE_FLAGS(pPage); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + // 1. Encode Header part + /* Encode kLen if need */ + if (pPage->kLen == TDB_VARIANT_LEN) { + nHeader += tdbPutVarInt(pCell + nHeader, kLen); + } + + /* Encode vLen if need */ + if (pPage->vLen == TDB_VARIANT_LEN) { + nHeader += tdbPutVarInt(pCell + nHeader, vLen); + } + + /* Encode SPgno if interior page */ + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; + nHeader = nHeader + sizeof(SPgno); + } + + // 2. Encode payload part + if (leaf) { + ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); + } else { + ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, NULL, 0, &nPayload); + } + if (ret < 0) { + // TODO: handle error + return -1; + } + + *szCell = nHeader + nPayload; + return 0; +} + +static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder *pDecoder) { + int nPayload; + + ASSERT(pDecoder->pKey == NULL); + + if (pDecoder->pVal) { + nPayload = pDecoder->kLen + pDecoder->vLen; + } else { + nPayload = pDecoder->kLen; + } + + if (nPayload <= pPage->maxLocal) { + // General case without overflow + pDecoder->pKey = (void *)pPayload; + if (!pDecoder->pVal) { + pDecoder->pVal = (void *)(pPayload + pDecoder->kLen); + } + } else { + // TODO: handle overflow case + ASSERT(0); + } + + return 0; +} + +static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { + u16 flags; + u8 leaf; + int nHeader; + int ret; + + nHeader = 0; + flags = TDB_PAGE_FLAGS(pPage); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + // Clear the state of decoder + pDecoder->kLen = -1; + pDecoder->pKey = NULL; + pDecoder->vLen = -1; + pDecoder->pVal = NULL; + pDecoder->pgno = 0; + + // 1. Decode header part + if (pPage->kLen == TDB_VARIANT_LEN) { + nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->kLen)); + } else { + pDecoder->kLen = pPage->kLen; + } + + if (pPage->vLen == TDB_VARIANT_LEN) { + nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->vLen)); + } else { + pDecoder->vLen = pPage->vLen; + } + + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; + pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); + nHeader = nHeader + sizeof(SPgno); + } + + // 2. Decode payload part + ret = tdbBtreeDecodePayload(pPage, pCell + nHeader, pDecoder); + if (ret < 0) { + return -1; + } + + return 0; +} + +#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c new file mode 100644 index 0000000000000000000000000000000000000000..00f38a19bbee7967d18069ed41ceb30baad32e10 --- /dev/null +++ b/source/libs/tdb/src/db/tdbDb.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct STDb { + STEnv *pEnv; + SBTree *pBt; +}; + +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb) { + STDb *pDb; + SPager *pPager; + int ret; + char fFullName[TDB_FILENAME_LEN]; + SPage *pPage; + SPgno pgno; + + *ppDb = NULL; + + pDb = (STDb *)calloc(1, sizeof(*pDb)); + if (pDb == NULL) { + return -1; + } + + // pDb->pEnv + pDb->pEnv = pEnv; + + pPager = tdbEnvGetPager(pEnv, fname); + if (pPager == NULL) { + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); + ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); + if (ret < 0) { + return -1; + } + } + + ASSERT(pPager != NULL); + + // pDb->pBt + ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt)); + if (ret < 0) { + return -1; + } + + *ppDb = pDb; + return 0; +} + +int tdbDbClose(STDb *pDb) { + // TODO + return 0; +} + +int tdbDbDrop(STDb *pDb) { + // TODO + return 0; +} + +int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { + SBtCursor btc; + SBtCursor *pCur; + int ret; + + pCur = &btc; + ret = tdbBtreeCursor(pCur, pDb->pBt); + if (ret < 0) { + return -1; + } + + ret = tdbBtCursorInsert(pCur, pKey, keyLen, pVal, valLen); + if (ret < 0) { + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index a415d6d60e500cae4389aaf79fdc18f60cae4349..9a4dcdbcd59a72b10740e3754339bb312be745d0 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -15,159 +15,56 @@ #include "tdbInt.h" -struct STDbEnv { - char * rootDir; // root directory of the environment - char * jname; // journal file name - TdFilePtr jpFile; // journal file fd - pgsz_t pgSize; // page size - cachesz_t cacheSize; // total cache size - STDbList dbList; // TDB List - SPgFileList pgfList; // SPgFile List - SPgCache * pPgCache; // page cache - struct { -#define TDB_ENV_PGF_HASH_BUCKETS 17 - SPgFileList buckets[TDB_ENV_PGF_HASH_BUCKETS]; - } pgfht; // page file hash table; -}; - -#define TDB_ENV_PGF_HASH(fileid) \ - ({ \ - uint8_t *tmp = (uint8_t *)(fileid); \ - tmp[0] + tmp[1] + tmp[2]; \ - }) - -static int tdbEnvDestroy(TENV *pEnv); - -int tdbEnvCreate(TENV **ppEnv, const char *rootDir) { - TENV * pEnv; - size_t slen; - size_t jlen; - - ASSERT(rootDir != NULL); +int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv) { + STEnv *pEnv; + int dsize; + int zsize; + u8 *pPtr; + int ret; *ppEnv = NULL; - slen = strlen(rootDir); - jlen = slen + strlen(TDB_JOURNAL_NAME) + 1; - pEnv = (TENV *)calloc(1, sizeof(*pEnv) + slen + 1 + jlen + 1); - if (pEnv == NULL) { - return -1; - } - - pEnv->rootDir = (char *)(&pEnv[1]); - pEnv->jname = pEnv->rootDir + slen + 1; - pEnv->jpFile = NULL; - pEnv->pgSize = TDB_DEFAULT_PGSIZE; - pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; - memcpy(pEnv->rootDir, rootDir, slen); - pEnv->rootDir[slen] = '\0'; - sprintf(pEnv->jname, "%s/%s", rootDir, TDB_JOURNAL_NAME); - - TD_DLIST_INIT(&(pEnv->dbList)); - TD_DLIST_INIT(&(pEnv->pgfList)); - - /* TODO */ - - *ppEnv = pEnv; - return 0; -} - -int tdbEnvOpen(TENV *pEnv) { - SPgCache *pPgCache; - int ret; - - ASSERT(pEnv != NULL); - - /* TODO: here we do not need to create the root directory, more - * work should be done here - */ - mkdir(pEnv->rootDir, 0755); - - ret = pgCacheOpen(&pPgCache, pEnv); - if (ret != 0) { - goto _err; - } - - pEnv->pPgCache = pPgCache; - return 0; - -_err: - return -1; -} - -int tdbEnvClose(TENV *pEnv) { - if (pEnv == NULL) return 0; - pgCacheClose(pEnv->pPgCache); - tdbEnvDestroy(pEnv); - return 0; -} + dsize = strlen(rootDir); + zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; -int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize) { - if (!TDB_IS_PGSIZE_VLD(pgSize) || cacheSize / pgSize < 10) { + pPtr = (uint8_t *)calloc(1, zsize); + if (pPtr == NULL) { return -1; } - /* TODO */ - - pEnv->pgSize = pgSize; - pEnv->cacheSize = cacheSize; - - return 0; -} - -pgsz_t tdbEnvGetPageSize(TENV *pEnv) { return pEnv->pgSize; } - -cachesz_t tdbEnvGetCacheSize(TENV *pEnv) { return pEnv->cacheSize; } - -SPgFile *tdbEnvGetPageFile(TENV *pEnv, const uint8_t fileid[]) { - SPgFileList *pBucket; - SPgFile * pPgFile; - - pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO - for (pPgFile = TD_DLIST_HEAD(pBucket); pPgFile != NULL; pPgFile = TD_DLIST_NODE_NEXT_WITH_FIELD(pPgFile, envHash)) { - if (memcmp(fileid, pPgFile->fileid, TDB_FILE_ID_LEN) == 0) break; - }; - - return pPgFile; -} - -SPgCache *tdbEnvGetPgCache(TENV *pEnv) { return pEnv->pPgCache; } - -static int tdbEnvDestroy(TENV *pEnv) { - // TODO - return 0; -} - -int tdbEnvBeginTxn(TENV *pEnv) { - pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); - if (pEnv->jpFile == NULL) { + pEnv = (STEnv *)pPtr; + pPtr += sizeof(*pEnv); + // pEnv->rootDir + pEnv->rootDir = pPtr; + memcpy(pEnv->rootDir, rootDir, dsize); + pEnv->rootDir[dsize] = '\0'; + pPtr = pPtr + dsize + 1; + // pEnv->jfname + pEnv->jfname = pPtr; + memcpy(pEnv->jfname, rootDir, dsize); + pEnv->jfname[dsize] = '/'; + memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); + pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; + + pEnv->jfd = -1; + + ret = tdbPCacheOpen(pageSize, cacheSize, &(pEnv->pCache)); + if (ret < 0) { return -1; } - return 0; -} + mkdir(rootDir, 0755); -int tdbEnvCommit(TENV *pEnv) { - /* TODO */ - taosCloseFile(&pEnv->jpFile); - pEnv->jpFile = NULL; + *ppEnv = pEnv; return 0; } -const char *tdbEnvGetRootDir(TENV *pEnv) { return pEnv->rootDir; } - -int tdbEnvRgstPageFile(TENV *pEnv, SPgFile *pPgFile) { - SPgFileList *pBucket; - - TD_DLIST_APPEND_WITH_FIELD(&(pEnv->pgfList), pPgFile, envPgfList); - - pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(pPgFile->fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO - TD_DLIST_APPEND_WITH_FIELD(pBucket, pPgFile, envHash); - +int tdbEnvClose(STEnv *pEnv) { + // TODO return 0; } -int tdbEnvRgstDB(TENV *pEnv, TDB *pDb) { +SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname) { // TODO - return 0; + return NULL; } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c new file mode 100644 index 0000000000000000000000000000000000000000..9d7181c1da292e9a9f5b242ebbc1235d3f2f0b00 --- /dev/null +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "tdbInt.h" + +struct SPCache { + int pageSize; + int cacheSize; + TdThreadMutex mutex; + int nFree; + SPage *pFree; + int nPage; + int nHash; + SPage **pgHash; + int nRecyclable; + SPage lru; +}; + +#define PCACHE_PAGE_HASH(pPgid) \ + ({ \ + u32 *t = (u32 *)((pPgid)->fileid); \ + t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; \ + }) +#define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL) + +// For page ref +#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0) +#if 0 +#define TDB_REF_PAGE(pPage) (++(pPage)->nRef) +#define TDB_UNREF_PAGE(pPage) (--(pPage)->nRef) +#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef) +#else +#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1) +#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1) +#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef)) +#endif + +static int tdbPCacheOpenImpl(SPCache *pCache); +static void tdbPCacheInitLock(SPCache *pCache); +static void tdbPCacheClearLock(SPCache *pCache); +static void tdbPCacheLock(SPCache *pCache); +static void tdbPCacheUnlock(SPCache *pCache); +static bool tdbPCacheLocked(SPCache *pCache); +static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); +static void tdbPCachePinPage(SPage *pPage); +static void tdbPCacheRemovePageFromHash(SPage *pPage); +static void tdbPCacheAddPageToHash(SPage *pPage); +static void tdbPCacheUnpinPage(SPage *pPage); +static void *tdbOsMalloc(void *arg, size_t size); +static void tdbOsFree(void *arg, void *ptr); + +int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { + SPCache *pCache; + void *pPtr; + SPage *pPgHdr; + + pCache = (SPCache *)calloc(1, sizeof(*pCache)); + if (pCache == NULL) { + return -1; + } + + pCache->pageSize = pageSize; + pCache->cacheSize = cacheSize; + + if (tdbPCacheOpenImpl(pCache) < 0) { + free(pCache); + return -1; + } + + *ppCache = pCache; + return 0; +} + +int tdbPCacheClose(SPCache *pCache) { + /* TODO */ + return 0; +} + +SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { + SPage *pPage; + + tdbPCacheLock(pCache); + + pPage = tdbPCacheFetchImpl(pCache, pPgid, alcNewPage); + if (pPage) { + TDB_REF_PAGE(pPage); + } + + tdbPCacheUnlock(pCache); + + return pPage; +} + +void tdbPCacheRelease(SPage *pPage) { + i32 nRef; + + nRef = TDB_UNREF_PAGE(pPage); + ASSERT(nRef >= 0); + + if (nRef == 0) { + if (1 /*TODO: page still clean*/) { + tdbPCacheUnpinPage(pPage); + } else { + // TODO + ASSERT(0); + } + } +} + +static void tdbPCacheInitLock(SPCache *pCache) { taosThreadMutexInit(&(pCache->mutex), NULL); } + +static void tdbPCacheClearLock(SPCache *pCache) { taosThreadMutexDestroy(&(pCache->mutex)); } + +static void tdbPCacheLock(SPCache *pCache) { taosThreadMutexLock(&(pCache->mutex)); } + +static void tdbPCacheUnlock(SPCache *pCache) { taosThreadMutexUnlock(&(pCache->mutex)); } + +static bool tdbPCacheLocked(SPCache *pCache) { + assert(0); + // TODO + return true; +} + +static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { + SPage *pPage; + + // 1. Search the hash table + pPage = pCache->pgHash[PCACHE_PAGE_HASH(pPgid) % pCache->nHash]; + while (pPage) { + if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break; + pPage = pPage->pHashNext; + } + + if (pPage || !alcNewPage) { + if (pPage) { + tdbPCachePinPage(pPage); + } + return pPage; + } + + // 2. Try to allocate a new page from the free list + if (pCache->pFree) { + pPage = pCache->pFree; + pCache->pFree = pPage->pFreeNext; + pCache->nFree--; + pPage->pLruNext = NULL; + } + + // 3. Try to Recycle a page + if (!pPage && !pCache->lru.pLruPrev->isAnchor) { + pPage = pCache->lru.pLruPrev; + tdbPCacheRemovePageFromHash(pPage); + tdbPCachePinPage(pPage); + } + + // 4. Try a stress allocation (TODO) + + // 5. Page here are just created from a free list + // or by recycling or allocated streesly, + // need to initialize it + if (pPage) { + memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); + pPage->pLruNext = NULL; + pPage->pPager = NULL; + tdbPCacheAddPageToHash(pPage); + } + + return pPage; +} + +static void tdbPCachePinPage(SPage *pPage) { + SPCache *pCache; + + pCache = pPage->pCache; + if (!PAGE_IS_PINNED(pPage)) { + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; + pPage->pLruNext = NULL; + + pCache->nRecyclable--; + } +} + +static void tdbPCacheUnpinPage(SPage *pPage) { + SPCache *pCache; + i32 nRef; + + pCache = pPage->pCache; + + tdbPCacheLock(pCache); + + nRef = TDB_GET_PAGE_REF(pPage); + ASSERT(nRef >= 0); + if (nRef == 0) { + // Add the page to LRU list + ASSERT(pPage->pLruNext == NULL); + + pPage->pLruPrev = &(pCache->lru); + pPage->pLruNext = pCache->lru.pLruNext; + pCache->lru.pLruNext->pLruPrev = pPage; + pCache->lru.pLruNext = pPage; + } + + pCache->nRecyclable++; + + tdbPCacheUnlock(pCache); +} + +static void tdbPCacheRemovePageFromHash(SPage *pPage) { + SPCache *pCache; + SPage **ppPage; + int h; + + pCache = pPage->pCache; + h = PCACHE_PAGE_HASH(&(pPage->pgid)); + for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) + ; + ASSERT(*ppPage == pPage); + *ppPage = pPage->pHashNext; + + pCache->nPage--; +} + +static void tdbPCacheAddPageToHash(SPage *pPage) { + SPCache *pCache; + int h; + + pCache = pPage->pCache; + h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; + + pPage->pHashNext = pCache->pgHash[h]; + pCache->pgHash[h] = pPage; + + pCache->nPage++; +} + +static int tdbPCacheOpenImpl(SPCache *pCache) { + SPage *pPage; + u8 *pPtr; + int tsize; + int ret; + + tdbPCacheInitLock(pCache); + + // Open the free list + pCache->nFree = 0; + pCache->pFree = NULL; + for (int i = 0; i < pCache->cacheSize; i++) { + ret = tdbPageCreate(pCache->pageSize, &pPage, tdbOsMalloc, NULL); + if (ret < 0) { + // TODO: handle error + return -1; + } + + // pPage->pgid = 0; + pPage->isAnchor = 0; + pPage->isLocalPage = 1; + pPage->pCache = pCache; + TDB_INIT_PAGE_REF(pPage); + pPage->pHashNext = NULL; + pPage->pLruNext = NULL; + pPage->pLruPrev = NULL; + pPage->pDirtyNext = NULL; + + pPage->pFreeNext = pCache->pFree; + pCache->pFree = pPage; + pCache->nFree++; + } + + // Open the hash table + pCache->nPage = 0; + pCache->nHash = pCache->cacheSize; + pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *)); + if (pCache->pgHash == NULL) { + // TODO + return -1; + } + + // Open LRU list + pCache->nRecyclable = 0; + pCache->lru.isAnchor = 1; + pCache->lru.pLruNext = &(pCache->lru); + pCache->lru.pLruPrev = &(pCache->lru); + + return 0; +} + +int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } + +static void *tdbOsMalloc(void *arg, size_t size) { + void *ptr; + + ptr = malloc(size); + + return ptr; +} + +static void tdbOsFree(void *arg, void *ptr) { free(ptr); } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c new file mode 100644 index 0000000000000000000000000000000000000000..df158de756a96fcc471a3b657ca1eaa4870dade0 --- /dev/null +++ b/source/libs/tdb/src/db/tdbPage.c @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +typedef struct __attribute__((__packed__)) { + u8 szCell[2]; + u8 nxOffset[2]; +} SFreeCell; + +typedef struct __attribute__((__packed__)) { + u8 szCell[3]; + u8 nxOffset[3]; +} SFreeCellL; + +/* For small page */ +#define TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCell *)(PCELL))->szCell) +#define TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCell *)(PCELL))->nxOffset) + +#define TDB_SPAGE_FREE_CELL_SIZE(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))[0] +#define TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))[0] + +#define TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) (TDB_SPAGE_FREE_CELL_SIZE(PCELL) = (SIZE)) +#define TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) (TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) = (OFFSET)) + +/* For large page */ +#define TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCellL *)(PCELL))->szCell) +#define TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCellL *)(PCELL))->nxOffset) + +#define TDB_LPAGE_FREE_CELL_SIZE(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL)) +#define TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) + +#define TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL), SIZE) +#define TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL), OFFSET) + +/* For page */ +#define TDB_PAGE_FREE_CELL_SIZE_PTR(PPAGE, PCELL) \ + (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) : TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL)) +#define TDB_PAGE_FREE_CELL_NXOFFSET_PTR(PPAGE, PCELL) \ + (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) + +#define TDB_PAGE_FREE_CELL_SIZE(PPAGE, PCELL) \ + (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE(PCELL) : TDB_SPAGE_FREE_CELL_SIZE(PCELL)) +#define TDB_PAGE_FREE_CELL_NXOFFSET(PPAGE, PCELL) \ + (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL)) + +#define TDB_PAGE_FREE_CELL_SIZE_SET(PPAGE, PCELL, SIZE) \ + do { \ + if (TDB_IS_LARGE_PAGE(PPAGE)) { \ + TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ + } else { \ + TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ + } \ + } while (0) +#define TDB_PAGE_FREE_CELL_NXOFFSET_SET(PPAGE, PCELL, OFFSET) \ + do { \ + if (TDB_IS_LARGE_PAGE(PPAGE)) { \ + TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ + } else { \ + TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ + } \ + } while (0) + +static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); +static int tdbPageDefragment(SPage *pPage); + +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { + SPage *pPage; + u8 *ptr; + int size; + + ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); + + *ppPage = NULL; + size = pageSize + sizeof(*pPage); + + ptr = (u8 *)((*xMalloc)(arg, size)); + if (pPage == NULL) { + return -1; + } + + memset(ptr, 0, size); + pPage = (SPage *)(ptr + pageSize); + + pPage->pData = ptr; + pPage->pageSize = pageSize; + if (pageSize < 65536) { + pPage->szOffset = 2; + pPage->szPageHdr = sizeof(SPageHdr); + pPage->szFreeCell = sizeof(SFreeCell); + } else { + pPage->szOffset = 3; + pPage->szPageHdr = sizeof(SPageHdrL); + pPage->szFreeCell = sizeof(SFreeCellL); + } + TDB_INIT_PAGE_LOCK(pPage); + + /* TODO */ + + *ppPage = pPage; + return 0; +} + +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { + u8 *ptr; + + ptr = pPage->pData; + (*xFree)(arg, ptr); + + return 0; +} + +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + int ret; + SCell *pTarget; + u8 *pTmp; + int j; + + if (pPage->nOverflow || szCell + pPage->szOffset > pPage->nFree) { + // TODO: need to figure out if pCell may be used by outside of this function + j = pPage->nOverflow++; + + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = idx; + } else { + ret = tdbPageAllocate(pPage, szCell, &pTarget); + if (ret < 0) { + return -1; + } + + memcpy(pTarget, pCell, szCell); + pTmp = pPage->pCellIdx + idx * pPage->szOffset; + memmove(pTmp + pPage->szOffset, pTmp, pPage->pFreeStart - pTmp - pPage->szOffset); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); + TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); + } + + return 0; +} + +int tdbPageDropCell(SPage *pPage, int idx) { + // TODO + return 0; +} + +static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { + SCell *pCell; + SFreeCell *pFreeCell; + u8 *pOffset; + int ret; + + ASSERT(pPage->nFree > size + pPage->szOffset); + + pCell = NULL; + *ppCell = NULL; + + // 1. Try to allocate from the free space area + if (pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset) { + pPage->pFreeEnd -= size; + pPage->pFreeStart += pPage->szOffset; + pCell = pPage->pFreeEnd; + } + + // 2. Try to allocate from the page free list + if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= pPage->szOffset) && TDB_PAGE_FCELL(pPage)) { + int szCell; + int nxOffset; + + pCell = pPage->pData + TDB_PAGE_FCELL(pPage); + pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell + : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); + szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); + nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + + for (;;) { + // Find a cell + if (szCell >= size) { + if (szCell - size >= pPage->szFreeCell) { + SCell *pTmpCell = pCell + size; + + TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); + TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); + // TODO: *pOffset = pTmpCell - pPage->pData; + } else { + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); + // TODO: *pOffset = nxOffset; + } + break; + } + + // Not find a cell yet + if (nxOffset > 0) { + pCell = pPage->pData + nxOffset; + pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); + szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); + nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + continue; + } else { + pCell = NULL; + break; + } + } + + if (pCell) { + pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; + } + } + + // 3. Try to dfragment and allocate again + if (pCell == NULL) { + ret = tdbPageDefragment(pPage); + if (ret < 0) { + return -1; + } + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset); + ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + + // Allocate from the free space area again + pPage->pFreeEnd -= size; + pPage->pFreeStart += pPage->szOffset; + pCell = pPage->pFreeEnd; + } + + ASSERT(pCell != NULL); + + pPage->nFree = pPage->nFree - size - pPage->szOffset; + *ppCell = pCell; + return 0; +} + +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { + // TODO + return 0; +} + +static int tdbPageDefragment(SPage *pPage) { + // TODO + ASSERT(0); + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c new file mode 100644 index 0000000000000000000000000000000000000000..ac0bc15e1d5ce768aa5a51931cef121765f4ae30 --- /dev/null +++ b/source/libs/tdb/src/db/tdbPager.c @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct SPager { + char *dbFileName; + char *jFileName; + int pageSize; + uint8_t fid[TDB_FILE_ID_LEN]; + int fd; + int jfd; + SPCache *pCache; + SPgno dbFileSize; + SPgno dbOrigSize; + int nDirty; + SPage *pDirty; + SPage *pDirtyTail; + u8 inTran; +}; + +typedef struct __attribute__((__packed__)) { + u8 hdrString[16]; + u16 pageSize; + SPgno freePage; + u32 nFreePages; + u8 reserved[102]; +} SFileHdr; + +TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); + +#define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL) + +static int tdbPagerReadPage(SPager *pPager, SPage *pPage); +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); +static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg); + +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { + uint8_t *pPtr; + SPager *pPager; + int fsize; + int zsize; + int ret; + + *ppPager = NULL; + + fsize = strlen(fileName); + zsize = sizeof(*pPager) /* SPager */ + + fsize + 1 /* dbFileName */ + + fsize + 8 + 1; /* jFileName */ + pPtr = (uint8_t *)calloc(1, zsize); + if (pPtr == NULL) { + return -1; + } + + pPager = (SPager *)pPtr; + pPtr += sizeof(*pPager); + // pPager->dbFileName + pPager->dbFileName = (char *)pPtr; + memcpy(pPager->dbFileName, fileName, fsize); + pPager->dbFileName[fsize] = '\0'; + pPtr += fsize + 1; + // pPager->jFileName + pPager->jFileName = (char *)pPtr; + memcpy(pPager->jFileName, fileName, fsize); + memcpy(pPager->jFileName + fsize, "-journal", 8); + pPager->jFileName[fsize + 8] = '\0'; + // pPager->pCache + pPager->pCache = pCache; + + pPager->fd = open(pPager->dbFileName, O_RDWR | O_CREAT, 0755); + if (pPager->fd < 0) { + return -1; + } + + ret = tdbGnrtFileID(pPager->dbFileName, pPager->fid, false); + if (ret < 0) { + return -1; + } + + pPager->jfd = -1; + pPager->pageSize = tdbPCacheGetPageSize(pCache); + + *ppPager = pPager; + return 0; +} + +int tdbPagerClose(SPager *pPager) { + // TODO + return 0; +} + +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) { + SPgno pgno; + SPage *pPage; + int ret; + + { + // TODO: try to search the main DB to get the page number + pgno = 0; + } + + // if (pgno == 0 && toCreate) { + // ret = tdbPagerAllocPage(pPager, &pPage, &pgno); + // if (ret < 0) { + // return -1; + // } + + // // TODO: Need to zero the page + + // ret = tdbPagerWrite(pPager, pPage); + // if (ret < 0) { + // return -1; + // } + // } + + *ppgno = pgno; + return 0; +} + +int tdbPagerWrite(SPager *pPager, SPage *pPage) { + int ret; + + if (pPager->inTran == 0) { + ret = tdbPagerBegin(pPager); + if (ret < 0) { + return -1; + } + } + + if (pPage->isDirty == 0) { + pPage->isDirty = 1; + // TODO: add the page to the dirty list + + // TODO: write the page to the journal + if (1 /*actually load from the file*/) { + } + } + return 0; +} + +int tdbPagerBegin(SPager *pPager) { + if (pPager->inTran) { + return 0; + } + + // Open the journal + pPager->jfd = open(pPager->jFileName, O_RDWR | O_CREAT, 0755); + if (pPager->jfd < 0) { + return -1; + } + + // TODO: write the size of the file + + pPager->inTran = 1; + + return 0; +} + +int tdbPagerCommit(SPager *pPager) { + // TODO + return 0; +} + +static int tdbPagerReadPage(SPager *pPager, SPage *pPage) { + i64 offset; + int ret; + + ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0); + + offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize); + ret = tdbPRead(pPager->fd, pPage->pData, pPager->pageSize, offset); + if (ret < 0) { + // TODO: handle error + return -1; + } + return 0; +} + +int tdbPagerGetPageSize(SPager *pPager) { return pPager->pageSize; } + +int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) { + SPage *pPage; + SPgid pgid; + int ret; + + // Fetch a page container from the page cache + memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); + pgid.pgno = pgno; + pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1); + if (pPage == NULL) { + return -1; + } + + // Initialize the page if need + if (!TDB_PAGE_INITIALIZED(pPage)) { + ret = tdbPagerInitPage(pPager, pPage, initPage, arg); + if (ret < 0) { + return -1; + } + } + + ASSERT(TDB_PAGE_INITIALIZED(pPage)); + ASSERT(pPage->pPager == pPager); + + *ppPage = pPage; + return 0; +} + +int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) { + int ret; + SPage *pPage; + SPgid pgid; + + // Allocate a page number + ret = tdbPagerAllocPage(pPager, ppgno); + if (ret < 0) { + return -1; + } + + ASSERT(*ppgno != 0); + + // Fetch a page container from the page cache + memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); + pgid.pgno = *ppgno; + pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1); + if (pPage == NULL) { + return -1; + } + + ASSERT(!TDB_PAGE_INITIALIZED(pPage)); + + // Initialize the page if need + ret = tdbPagerInitPage(pPager, pPage, initPage, arg); + if (ret < 0) { + return -1; + } + + ASSERT(TDB_PAGE_INITIALIZED(pPage)); + ASSERT(pPage->pPager == pPager); + + *ppPage = pPage; + return 0; +} + +static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { + // TODO: Allocate a page from the free list + return 0; +} + +static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) { + *ppgno = ++pPager->dbFileSize; + return 0; +} + +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { + int ret; + + *ppgno = 0; + + // Try to allocate from the free list of the pager + ret = tdbPagerAllocFreePage(pPager, ppgno); + if (ret < 0) { + return -1; + } + + if (*ppgno != 0) return 0; + + // Allocate the page by extending the pager + ret = tdbPagerAllocNewPage(pPager, ppgno); + if (ret < 0) { + return -1; + } + + ASSERT(*ppgno != 0); + + return 0; +} + +static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg) { + int ret; + int lcode; + int nLoops; + + lcode = TDB_TRY_LOCK_PAGE(pPage); + if (lcode == P_LOCK_SUCC) { + if (TDB_PAGE_INITIALIZED(pPage)) { + TDB_UNLOCK_PAGE(pPage); + return 0; + } + + ret = (*initPage)(pPage, arg); + if (ret < 0) { + TDB_UNLOCK_PAGE(pPage); + return -1; + } + + pPage->pPager = pPager; + + TDB_UNLOCK_PAGE(pPage); + } else if (lcode == P_LOCK_BUSY) { + nLoops = 0; + for (;;) { + if (TDB_PAGE_INITIALIZED(pPage)) break; + nLoops++; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + } else { + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPgCache.c b/source/libs/tdb/src/db/tdbPgCache.c deleted file mode 100644 index 1b000aa6d6f85dd983b0d60a3c2565990cfd815e..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/db/tdbPgCache.c +++ /dev/null @@ -1,580 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "tdbInt.h" - -typedef TD_DLIST(SPage) SPgList; -struct SPgCache { - TENV * pEnv; // TENV containing this page cache - pgsz_t pgsize; - int32_t npage; - SPage **pages; - SPgList freeList; - SPgList lru; - struct { - int32_t nbucket; - SPgList *buckets; - } pght; // page hash table -}; - -static void pgCachePinPage(SPage *pPage); -static void pgCacheUnpinPage(SPage *pPage); - -int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv) { - SPgCache *pPgCache; - SPage * pPage; - void * pData; - pgsz_t pgSize; - cachesz_t cacheSize; - int32_t npage; - int32_t nbucket; - size_t msize; - - *ppPgCache = NULL; - pgSize = tdbEnvGetPageSize(pEnv); - cacheSize = tdbEnvGetCacheSize(pEnv); - npage = cacheSize / pgSize; - nbucket = npage; - msize = sizeof(*pPgCache) + sizeof(SPage *) * npage + sizeof(SPgList) * nbucket; - - // Allocate the handle - pPgCache = (SPgCache *)calloc(1, msize); - if (pPgCache == NULL) { - return -1; - } - - // Init the handle - pPgCache->pEnv = pEnv; - pPgCache->pgsize = pgSize; - pPgCache->npage = npage; - pPgCache->pages = (SPage **)(&pPgCache[1]); - pPgCache->pght.nbucket = nbucket; - pPgCache->pght.buckets = (SPgList *)(&(pPgCache->pages[npage])); - - TD_DLIST_INIT(&(pPgCache->freeList)); - - for (int32_t i = 0; i < npage; i++) { - pData = malloc(pgSize + sizeof(SPage)); - if (pData == NULL) { - return -1; - // TODO: handle error - } - - pPage = POINTER_SHIFT(pData, pgSize); - - pPage->pgid = TDB_IVLD_PGID; - pPage->frameid = i; - pPage->pData = pData; - - // add current page to the page cache - pPgCache->pages[i] = pPage; - TD_DLIST_APPEND_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode); - } - -#if 0 - for (int32_t i = 0; i < nbucket; i++) { - TD_DLIST_INIT(pPgCache->pght.buckets + i); - } -#endif - - *ppPgCache = pPgCache; - return 0; -} - -int pgCacheClose(SPgCache *pPgCache) { - SPage *pPage; - if (pPgCache) { - for (int32_t i = 0; i < pPgCache->npage; i++) { - pPage = pPgCache->pages[i]; - tfree(pPage->pData); - } - - free(pPgCache); - } - - return 0; -} - -#define PG_CACHE_HASH(fileid, pgno) \ - ({ \ - uint64_t *tmp = (uint64_t *)(fileid); \ - (tmp[0] + tmp[1] + tmp[2] + (pgno)); \ - }) - -SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid) { - SPage * pPage; - SPgFile *pPgFile; - SPgList *pBucket; - - // 1. Search the page hash table SPgCache.pght - pBucket = pPgCache->pght.buckets + (PG_CACHE_HASH(pgid.fileid, pgid.pgno) % pPgCache->pght.nbucket); - pPage = TD_DLIST_HEAD(pBucket); - while (pPage && tdbCmprPgId(&(pPage->pgid), &pgid)) { - pPage = TD_DLIST_NODE_NEXT_WITH_FIELD(pPage, pghtNode); - } - - if (pPage) { - // Page is found, pin the page and return the page - pgCachePinPage(pPage); - return pPage; - } - - // 2. Check the free list - pPage = TD_DLIST_HEAD(&(pPgCache->freeList)); - if (pPage) { - TD_DLIST_POP_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode); - pgCachePinPage(pPage); - return pPage; - } - - // 3. Try to recycle a page from the LRU list - pPage = TD_DLIST_HEAD(&(pPgCache->lru)); - if (pPage) { - TD_DLIST_POP_WITH_FIELD(&(pPgCache->lru), pPage, lruNode); - // TODO: remove from the hash table - pgCachePinPage(pPage); - return pPage; - } - - // 4. If a memory allocator is set, try to allocate from the allocator (TODO) - - return NULL; -} - -int pgCacheRelease(SPage *pPage) { - // TODO - return 0; -} - -static void pgCachePinPage(SPage *pPage) { - // TODO -} - -static void pgCacheUnpinPage(SPage *pPage) { - // TODO -} - -#if 0 -// Exposed handle -typedef struct TDB_MPOOL TDB_MPOOL; -typedef struct TDB_MPFILE TDB_MPFILE; - -typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; -typedef struct pg_t { - SRWLatch rwLatch; - frame_id_t frameid; - pgid_t pgid; - uint8_t dirty; - uint8_t rbit; - int32_t pinRef; - pg_free_dlist_node_t free; - pg_hash_dlist_node_t hash; - void * p; -} pg_t; - -typedef TD_DLIST(pg_t) pg_list_t; -typedef struct { - SRWLatch latch; - TD_DLIST(TDB_MPFILE); -} mpf_bucket_t; -struct TDB_MPOOL { - int64_t cachesize; - pgsz_t pgsize; - int32_t npages; - pg_t * pages; - pg_list_t freeList; - frame_id_t clockHand; - struct { - int32_t nbucket; - pg_list_t *hashtab; - } pgtab; // page table, hash - struct { -#define MPF_HASH_BUCKETS 16 - mpf_bucket_t buckets[MPF_HASH_BUCKETS]; - } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table -}; - -#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] - -typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; -struct TDB_MPFILE { - char * fname; // file name - int fd; // fd - uint8_t fileid[TDB_FILE_ID_LEN]; // file ID - TDB_MPOOL * mp; // underlying memory pool - td_mpf_dlist_node_t node; -}; - -/*=================================================== Exposed apis ==================================================*/ -// TDB_MPOOL -int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize); -int tdbMPoolClose(TDB_MPOOL *mp); -int tdbMPoolSync(TDB_MPOOL *mp); - -// TDB_MPFILE -int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); -int tdbMPoolFileClose(TDB_MPFILE *mpf); -int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); -int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); -int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFileSync(TDB_MPFILE *mpf); - -static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); -static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); -static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); -static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); -static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); -static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); - -int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) { - TDB_MPOOL *mp = NULL; - size_t tsize; - pg_t * pagep; - - // check parameters - if (!TDB_IS_PGSIZE_VLD(pgsize)) { - tdbError("invalid page size"); - return -1; - } - - // allocate handle - mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); - if (mp == NULL) { - tdbError("failed to malloc memory pool handle"); - goto _err; - } - - // initialize the handle - mp->cachesize = cachesize; - mp->pgsize = pgsize; - mp->npages = cachesize / pgsize; - mp->clockHand = 0; - - TD_DLIST_INIT(&mp->freeList); - - mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t)); - if (mp->pages == NULL) { - tdbError("failed to malloc memory pool pages"); - goto _err; - } - - for (frame_id_t i = 0; i < mp->npages; i++) { - mp->pages[i].p = malloc(pgsize); - if (mp->pages[i].p == NULL) { - goto _err; - } - - taosInitRWLatch(&mp->pages[i].rwLatch); - mp->pages[i].frameid = i; - mp->pages[i].pgid = TDB_IVLD_PGID; - - // add new page to the free list - TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free); - } - -#define PGTAB_FACTOR 1.0 - mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR; - mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t)); - if (mp->pgtab.hashtab == NULL) { - tdbError("failed to malloc memory pool hash table"); - goto _err; - } - - // return - *mpp = mp; - return 0; - -_err: - tdbMPoolClose(mp); - *mpp = NULL; - return -1; -} - -int tdbMPoolClose(TDB_MPOOL *mp) { - if (mp) { - tfree(mp->pgtab.hashtab); - if (mp->pages) { - for (int i = 0; i < mp->npages; i++) { - tfree(mp->pages[i].p); - } - - free(mp->pages); - } - - free(mp); - } - return 0; -} - -int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { - TDB_MPFILE *mpf; - - if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) { - return -1; - } - - mpf->fd = -1; - - if ((mpf->fname = strdup(fname)) == NULL) { - goto _err; - } - - if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) { - goto _err; - } - - if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) { - goto _err; - } - - // Register current MPF to MP - tdbMPoolRegFile(mp, mpf); - - *mpfp = mpf; - return 0; - -_err: - tdbMPoolFileClose(mpf); - *mpfp = NULL; - return -1; -} - -int tdbMPoolFileClose(TDB_MPFILE *mpf) { - if (mpf) { - if (mpf->fd > 0) { - close(mpf->fd); - } - tfree(mpf->fname); - free(mpf); - } - return 0; -} - -#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \ - ({ \ - uint64_t *tmp = (uint64_t *)fileid; \ - (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ - }) - -int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) { - // TODO - return 0; -} - -int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) { - // TODO - return 0; -} - -int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { - pg_t * pagep; - TDB_MPOOL *mp; - pg_list_t *pglist; - - mp = mpf->mp; - - // check if the page already in pool - pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); - pagep = TD_DLIST_HEAD(pglist); - while (pagep) { - if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) { - break; - } - - pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash); - } - - if (pagep) { - // page is found - // todo: pin the page and return - *(void **)addr = pagep->p; - return 0; - } - - // page not found - pagep = TD_DLIST_HEAD(&mp->freeList); - if (pagep) { - // has free page - TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); - } else { - // no free page available - tdbMPoolClockEvictPage(mp, &pagep); - if (pagep) { - if (pagep->dirty) { - // TODO: Handle dirty page eviction - } - } - } - - if (pagep == NULL) { - // no available container page - return -1; - } - - // load page from the disk if a container page is available - // TODO: load the page from the disk - if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) { - return -1; - } - - memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN); - pagep->pgid.pgno = pgno; - pagep->dirty = 0; - pagep->pinRef = 1; - - // add current page to page table - TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); - - return 0; -} - -int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { - // TODO - return 0; -} - -#define MPF_GET_BUCKETID(fileid) \ - ({ \ - uint64_t *tmp = (uint64_t *)fileid; \ - (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ - }) - -static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - mpf_bucket_t *bktp; - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); - - taosWLockLatch(&(bktp->latch)); - - TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node); - - taosWUnLockLatch(&(bktp->latch)); - - mpf->mp = mp; -} - -static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { - TDB_MPFILE * mpf = NULL; - mpf_bucket_t *bktp; - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid); - - taosRLockLatch(&(bktp->latch)); - - mpf = TD_DLIST_HEAD(bktp); - while (mpf) { - if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) { - break; - } - - mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node); - } - - taosRUnLockLatch(&(bktp->latch)); - - return mpf; -} - -static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - mpf_bucket_t *bktp; - TDB_MPFILE * tmpf; - - if (mpf->mp == NULL) return; - - ASSERT(mpf->mp == mp); - - bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); - - taosWLockLatch(&(bktp->latch)); - - tmpf = TD_DLIST_HEAD(bktp); - - while (tmpf) { - if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) { - TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node); - break; - } - - tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node); - } - - taosWUnLockLatch(&(bktp->latch)); - - ASSERT(tmpf == mpf); -} - -static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { - pgsz_t pgsize; - TDB_MPOOL *mp; - off_t offset; - size_t rsize; - - mp = mpf->mp; - pgsize = mp->pgsize; - offset = pgno * pgsize; - - // TODO: use loop to read all data - rsize = pread(mpf->fd, p, pgsize, offset); - // TODO: error handle - - return 0; -} - -static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { - pgsz_t pgsize; - TDB_MPOOL *mp; - off_t offset; - - mp = mpf->mp; - pgsize = mp->pgsize; - offset = pgno * pgsize; - - lseek(mpf->fd, offset, SEEK_SET); - // TODO: handle error - - write(mpf->fd, p, pgsize); - // TODO: handle error - - return 0; -} - -static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { - pg_t * pagep; - frame_id_t och; - - *pagepp = NULL; - och = mp->clockHand; - - do { - pagep = mp->pages + mp->clockHand; - mp->clockHand = (mp->clockHand + 1) % mp->npages; - - if (pagep->pinRef == 0) { - if (pagep->rbit == 1) { - pagep->rbit = 0; - } else { - break; - } - } - - if (mp->clockHand == och) { - return; - } - } while (1); - - *pagepp = pagep; -} - -#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPgFile.c b/source/libs/tdb/src/db/tdbPgFile.c deleted file mode 100644 index 12f062ebf767b927172e91a7a86de4e3cae7d32e..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/db/tdbPgFile.c +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -typedef struct SPage1 { - char magic[64]; - pgno_t mdbRootPgno; // master DB root page number - pgno_t freePgno; // free list page number - uint32_t nFree; // number of free pages -} SPage1; - -typedef struct SFreePage { - /* TODO */ -} SFreePage; - -TDB_STATIC_ASSERT(sizeof(SPage1) <= TDB_MIN_PGSIZE, "TDB Page1 definition too large"); - -static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData); - -int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { - SPgFile * pPgFile; - SPgCache *pPgCache; - size_t fnameLen; - pgno_t fsize; - - *ppPgFile = NULL; - - // create the handle - fnameLen = strlen(fname); - pPgFile = (SPgFile *)calloc(1, sizeof(*pPgFile) + fnameLen + 1); - if (pPgFile == NULL) { - return -1; - } - - ASSERT(pEnv != NULL); - - // init the handle - pPgFile->fname = (char *)(&(pPgFile[1])); - memcpy(pPgFile->fname, fname, fnameLen); - pPgFile->fname[fnameLen] = '\0'; - pPgFile->pFile = NULL; - - pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); - if (pPgFile->pFile == NULL) { - // TODO: handle error - return -1; - } - - tdbGnrtFileID(fname, pPgFile->fileid, false); - tdbGetFileSize(fname, tdbEnvGetPageSize(pEnv), &fsize); - - pPgFile->fsize = fsize; - pPgFile->lsize = fsize; - - if (pPgFile->fsize == 0) { - // A created file - pgno_t pgno; - pgid_t pgid; - - pgFileAllocatePage(pPgFile, &pgno); - - ASSERT(pgno == 1); - - memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN); - pgid.pgno = pgno; - - pgCacheFetch(pPgCache, pgid); - // Need to allocate the first page as a description page - } else { - // An existing file - } - - /* TODO: other open operations */ - - // add the page file to the environment - tdbEnvRgstPageFile(pEnv, pPgFile); - pPgFile->pEnv = pEnv; - - *ppPgFile = pPgFile; - return 0; -} - -int pgFileClose(SPgFile *pPgFile) { - if (pPgFile) { - if (pPgFile->pFile != NULL) { - taosCloseFile(&pPgFile->pFile); - } - - tfree(pPgFile->fname); - free(pPgFile); - } - - return 0; -} - -SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno) { - SPgCache *pPgCache; - SPage * pPage; - pgid_t pgid; - - // 1. Fetch from the page cache - // pgCacheFetch(pPgCache, pgid); - - // 2. If only get a page frame, no content, maybe - // need to load from the file - if (1 /*page not initialized*/) { - if (pgno < pPgFile->fsize) { - // load the page content from the disk - // ?? How about the freed pages ?? - } else { - // zero the page, make the page as a empty - // page with zero records. - } - } - -#if 0 - pPgCache = pPgFile->pPgCache; - pPage = NULL; - memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN); - pgid.pgno = pgno; - - if (pgno > pPgFile->pgFileSize) { - // TODO - } else { - pPage = pgCacheFetch(pPgCache, pgid); - if (1 /*Page is cached, no need to load from file*/) { - return pPage; - } else { - // TODO: handle error - if (pgFileRead(pPgFile, pgno, (void *)pPage) < 0) { - // todoerr - } - return pPage; - } - } -#endif - - return pPage; -} - -int pgFileRelease(SPage *pPage) { - pgCacheRelease(pPage); - return 0; -} - -int pgFileWrite(SPage *pPage) { - // TODO - return 0; -} - -int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno) { - pgno_t pgno; - SPage1 * pPage1; - SPgCache *pPgCache; - pgid_t pgid; - SPage * pPage; - - if (pPgFile->lsize == 0) { - pgno = ++(pPgFile->lsize); - } else { - if (0) { - // TODO: allocate from the free list - pPage = pgCacheFetch(pPgCache, pgid); - - if (pPage1->nFree > 0) { - // TODO - } else { - pgno = ++(pPgFile->lsize); - } - } else { - pgno = ++(pPgFile->lsize); - } - } - - *pPgno = pgno; - return 0; -} - -static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) { - pgsz_t pgSize; - ssize_t rsize; - uint8_t *pTData; - size_t szToRead; - -#if 0 - - // pgSize = ; (TODO) - pTData = pData; - szToRead = pgSize; - for (; szToRead > 0;) { - rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize); - if (rsize < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (rsize == 0) { - return -1; - } - - szToRead -= rsize; - pTData += rsize; - } -#endif - - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 237a39e47db8223827320e070238e7b251d3bcc0..b5373be9dd6010550393a23402ba99ac8a95a9fe 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -51,7 +51,8 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { // return access(pathname, flags); // } -int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { +int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize) { + struct stat st; int ret; int64_t file_size = 0; ret = taosStatFile(fname, &file_size, NULL); @@ -63,4 +64,29 @@ int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { *pSize = file_size / pgSize; return 0; +} + +int tdbPRead(int fd, void *pData, int count, i64 offset) { + void *pBuf; + int nbytes; + i64 ioffset; + int iread; + + pBuf = pData; + nbytes = count; + ioffset = offset; + while (nbytes > 0) { + iread = pread(fd, pBuf, nbytes, ioffset); + if (iread < 0) { + /* TODO */ + } else if (iread == 0) { + return (count - iread); + } + + nbytes = nbytes - iread; + pBuf = (void *)((u8 *)pBuf + iread); + ioffset += iread; + } + + return count; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index 94af3331bae7b5485026ffe5bb88c35ec65f8d5a..c1fe77c22ea18ad0720f74659f7eac46259c74f3 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -23,20 +23,21 @@ extern "C" { typedef struct SBTree SBTree; typedef struct SBtCursor SBtCursor; -// SBTree -int btreeOpen(SBTree **ppBt, SPgFile *pPgFile); -int btreeClose(SBTree *pBt); - -// SBtCursor -int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt); -int btreeCursorClose(SBtCursor *pBtCur); -int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey); -int btreeCursorNext(SBtCursor *pBtCur); - -struct SBTree { - pgno_t root; +struct SBtCursor { + SBTree *pBt; + i8 iPage; + SPage *pPage; + int idx; + int idxStack[BTREE_MAX_DEPTH + 1]; + SPage *pgStack[BTREE_MAX_DEPTH + 1]; + void *pBuf; }; +int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); +int tdbBtreeClose(SBTree *pBt); +int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); +int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/inc/dndBnode.h b/source/libs/tdb/src/inc/tdbDb.h similarity index 64% rename from source/dnode/mgmt/impl/inc/dndBnode.h rename to source/libs/tdb/src/inc/tdbDb.h index 080cd2e487c2ca74bb3151742b4eed91a28040b3..06ea74a83ee0642031364b6c1d7f6d1a445fe669 100644 --- a/source/dnode/mgmt/impl/inc/dndBnode.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -13,23 +13,22 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_BNODE_H_ -#define _TD_DND_BNODE_H_ +#ifndef _TD_TDB_DB_H_ +#define _TD_TDB_DB_H_ #ifdef __cplusplus extern "C" { #endif -#include "dndEnv.h" -int32_t dndInitBnode(SDnode *pDnode); -void dndCleanupBnode(SDnode *pDnode); +typedef struct STDb STDb; -void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); -int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); +int tdbDbClose(STDb *pDb); +int tdbDbDrop(STDb *pDb); +int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); #ifdef __cplusplus } #endif -#endif /*_TD_DND_BNODE_H_*/ \ No newline at end of file +#endif /*_TD_TDB_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbEnv.h b/source/libs/tdb/src/inc/tdbEnv.h index 6cb5c7a2cda8537ee07a4820e9cb9d5ae6df93a9..959b963a07769674e5d28979bfcb6183dde8318e 100644 --- a/source/libs/tdb/src/inc/tdbEnv.h +++ b/source/libs/tdb/src/inc/tdbEnv.h @@ -20,11 +20,17 @@ extern "C" { #endif -const char* tdbEnvGetRootDir(TENV* pEnv); -SPgFile* tdbEnvGetPageFile(TENV* pEnv, const uint8_t fileid[]); -SPgCache* tdbEnvGetPgCache(TENV* pEnv); -int tdbEnvRgstPageFile(TENV* pEnv, SPgFile* pPgFile); -int tdbEnvRgstDB(TENV* pEnv, TDB* pDb); +typedef struct STEnv { + char * rootDir; + char * jfname; + int jfd; + SPCache *pCache; +} STEnv; + +int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv); +int tdbEnvClose(STEnv *pEnv); + +SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index ac42e1500293f1c74fec9d6d89c253d772198fc5..5902a6a71630402fbe681830e394ce95df304ab0 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -19,16 +19,33 @@ #include "tlist.h" #include "tlockfree.h" -#include "tdb.h" +// #include "tdb.h" #ifdef __cplusplus extern "C" { #endif -typedef struct SPgFile SPgFile; +typedef int8_t i8; +typedef int16_t i16; +typedef int32_t i32; +typedef int64_t i64; +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; + +// p must be u8 * +#define TDB_GET_U24(p) ((p)[0] * 65536 + *(u16 *)((p) + 1)) +#define TDB_PUT_U24(p, v) \ + do { \ + int tv = (v); \ + (p)[2] = tv & 0xff; \ + (p)[1] = (tv >> 8) & 0xff; \ + (p)[0] = (tv >> 16) & 0xff; \ + } while (0) -// pgno_t -typedef int32_t pgno_t; +// SPgno +typedef u32 SPgno; #define TDB_IVLD_PGNO ((pgno_t)0) // fileid @@ -37,8 +54,8 @@ typedef int32_t pgno_t; // pgid_t typedef struct { uint8_t fileid[TDB_FILE_ID_LEN]; - pgno_t pgno; -} pgid_t; + SPgno pgno; +} pgid_t, SPgid; #define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; @@ -61,18 +78,14 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { } } -// framd_id_t -typedef int32_t frame_id_t; +#define TDB_IS_SAME_PAGE(pPgid1, pPgid2) (tdbCmprPgId(pPgid1, pPgid2) == 0) // pgsz_t -#define TDB_MIN_PGSIZE 512 -#define TDB_MAX_PGSIZE 65536 -#define TDB_DEFAULT_PGSIZE 4096 +#define TDB_MIN_PGSIZE 512 // 512B +#define TDB_MAX_PGSIZE 16777216 // 16M +#define TDB_DEFAULT_PGSIZE 4096 #define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) -// pgoff_t -typedef pgsz_t pgoff_t; - // cache #define TDB_DEFAULT_CACHE_SIZE (256 * 4096) // 1M @@ -100,7 +113,7 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; } \ } while (0) -#define TDB_VARIANT_LEN (int)-1 +#define TDB_VARIANT_LEN ((int)-1) // page payload format // + + [key] + [value] @@ -115,18 +128,40 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; /* TODO */ \ } while (0) +typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); + #define TDB_JOURNAL_NAME "tdb.journal" +#define TDB_FILENAME_LEN 128 + +#define TDB_DEFAULT_FANOUT 6 + +#define BTREE_MAX_DEPTH 20 + +#define TDB_FLAG_IS(flags, flag) ((flags) == (flag)) +#define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0) +#define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0) +#define TDB_FLAG_ADD(flags, flag) ((flags) |= (flag)) +#define TDB_FLAG_REMOVE(flags, flag) ((flags) &= (~(flag))) + +typedef struct SPager SPager; +typedef struct SPCache SPCache; +typedef struct SPage SPage; + #include "tdbUtil.h" -#include "tdbBtree.h" +#include "tdbPCache.h" -#include "tdbPgCache.h" +#include "tdbPager.h" -#include "tdbPgFile.h" +#include "tdbBtree.h" #include "tdbEnv.h" +#include "tdbDb.h" + +#include "tdbPage.h" + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbPgCache.h b/source/libs/tdb/src/inc/tdbPCache.h similarity index 54% rename from source/libs/tdb/src/inc/tdbPgCache.h rename to source/libs/tdb/src/inc/tdbPCache.h index c25ef27c1050dd83495adf4e50215a61810c5825..ff4f1acbb6cf8091e083202fb6fdd19477957d01 100644 --- a/source/libs/tdb/src/inc/tdbPgCache.h +++ b/source/libs/tdb/src/inc/tdbPCache.h @@ -20,26 +20,25 @@ extern "C" { #endif -typedef struct SPgCache SPgCache; -typedef struct SPage SPage; - -// SPgCache -int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv); -int pgCacheClose(SPgCache *pPgCache); - -SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid); -int pgCacheRelease(SPage *pPage); - -// SPage -typedef TD_DLIST_NODE(SPage) SPgListNode; -struct SPage { - pgid_t pgid; // page id - frame_id_t frameid; // frame id - uint8_t * pData; // real data - SPgListNode freeNode; // for SPgCache.freeList - SPgListNode pghtNode; // for pght - SPgListNode lruNode; // for LRU -}; +#define TDB_PCACHE_PAGE \ + u8 isAnchor; \ + u8 isLocalPage; \ + u8 isDirty; \ + i32 nRef; \ + SPCache *pCache; \ + SPage *pFreeNext; \ + SPage *pHashNext; \ + SPage *pLruNext; \ + SPage *pLruPrev; \ + SPage *pDirtyNext; \ + SPager *pPager; \ + SPgid pgid; + +int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); +int tdbPCacheClose(SPCache *pCache); +SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); +void tdbPCacheRelease(SPage *pPage); +int tdbPCacheGetPageSize(SPCache *pCache); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h new file mode 100644 index 0000000000000000000000000000000000000000..8a51c331b62b868f45c754109f9cabb3ba358d0a --- /dev/null +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_PAGE_H_ +#define _TDB_PAGE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef u8 SCell; + +// PAGE APIS implemented +typedef struct { + int szOffset; + int szPageHdr; + int szFreeCell; + // flags + u16 (*getFlags)(SPage *); + void (*setFlags)(SPage *, u16); + // cell number + int (*getCellNum)(SPage *); + void (*setCellNum)(SPage *, int); + // cell content offset + int (*getCellBody)(SPage *); + void (*setCellBody)(SPage *, int); + // first free cell offset (0 means no free cells) + int (*getCellFree)(SPage *); + void (*setCellFree)(SPage *, int); + // total free bytes + int (*getFreeBytes)(SPage *); + void (*setFreeBytes)(SPage *, int); + // cell offset at idx + int (*getCellOffset)(SPage *, int); + void (*setCellOffset)(SPage *, int, int); +} SPageMethods; + +// Page footer +typedef struct __attribute__((__packed__)) { + u8 cksm[4]; +} SPageFtr; + +struct SPage { + TdThreadSpinlock lock; + u8 *pData; + int pageSize; + SPageMethods *pPageMethods; + // Fields below used by pager and am + u8 szAmHdr; + u8 *pPageHdr; + u8 *pAmHdr; + u8 *pCellIdx; + u8 *pFreeStart; + u8 *pFreeEnd; + SPageFtr *pPageFtr; + int kLen; // key length of the page, -1 for unknown + int vLen; // value length of the page, -1 for unknown + int nFree; + int maxLocal; + int minLocal; + int nOverflow; + SCell *apOvfl[4]; + int aiOvfl[4]; + // Fields used by SPCache + TDB_PCACHE_PAGE +}; + +/* For page */ +#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) +#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) +#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) +#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) +#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) +#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) + +#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) +#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) +#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) +#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) +#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) +#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) + +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) + +#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) + +// For page lock +#define P_LOCK_SUCC 0 +#define P_LOCK_BUSY 1 +#define P_LOCK_FAIL -1 + +#define TDB_INIT_PAGE_LOCK(pPage) taosThreadSpinInit(&((pPage)->lock), 0) +#define TDB_DESTROY_PAGE_LOCK(pPage) taosThreadSpinDestroy(&((pPage)->lock)) +#define TDB_LOCK_PAGE(pPage) taosThreadSpinLock(&((pPage)->lock)) +#define TDB_UNLOCK_PAGE(pPage) taosThreadSpinUnlock(&((pPage)->lock)) +#define TDB_TRY_LOCK_PAGE(pPage) \ + ({ \ + int ret; \ + if (pthread_spin_trylock(&((pPage)->lock)) == 0) { \ + ret = P_LOCK_SUCC; \ + } else if (errno == EBUSY) { \ + ret = P_LOCK_BUSY; \ + } else { \ + ret = P_LOCK_FAIL; \ + } \ + ret; \ + }) + +// APIs +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); +int tdbPageDropCell(SPage *pPage, int idx); + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_PAGE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h new file mode 100644 index 0000000000000000000000000000000000000000..e4ed8552fd411a8db7f8d9a91f98703539486d14 --- /dev/null +++ b/source/libs/tdb/src/inc/tdbPager.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_PAGER_H_ +#define _TDB_PAGER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); +int tdbPagerClose(SPager *pPager); +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); +int tdbPagerWrite(SPager *pPager, SPage *pPage); +int tdbPagerBegin(SPager *pPager); +int tdbPagerCommit(SPager *pPager); +int tdbPagerGetPageSize(SPager *pPager); +int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_PAGER_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h deleted file mode 100644 index eaeebf6b9d3d6260b181b0e8fd9607a2befe7482..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbPgFile.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_PAGE_FILE_H_ -#define _TD_PAGE_FILE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct __attribute__((__packed__)) { - char hdrInfo[16]; // info string - pgsz_t szPage; // page size of current file - int32_t cno; // commit number counter - pgno_t freePgno; // freelist page number - uint8_t resv[100]; // reserved space -} SPgFileHdr; - -#define TDB_PG_FILE_HDR_SIZE 128 - -TDB_STATIC_ASSERT(sizeof(SPgFileHdr) == TDB_PG_FILE_HDR_SIZE, "Page file header size if not 128"); - -struct SPgFile { - TENV * pEnv; // env containing this page file - char * fname; // backend file name - uint8_t fileid[TDB_FILE_ID_LEN]; // file id - pgno_t lsize; // page file logical size (for count) - pgno_t fsize; // real file size on disk (for rollback) - TdFilePtr pFile; - SPgFileListNode envHash; - SPgFileListNode envPgfList; -}; - -int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv); -int pgFileClose(SPgFile *pPgFile); - -SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno); -int pgFileRelease(SPage *pPage); - -int pgFileWrite(SPage *pPage); -int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_PAGE_FILE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index ca05790f770556dd3fc485e70a6e49fc4e43af38..8aaded933a02aa3fc9d9ab843ce546d70b6ff45f 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -35,7 +35,48 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); // #define TDB_W_OK 0x4 // int tdbCheckFileAccess(const char *pathname, int mode); -int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize); +int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); + +int tdbPRead(int fd, void *pData, int count, i64 offset); + +static inline int tdbPutVarInt(u8 *p, int v) { + int n = 0; + + for (;;) { + if (v <= 0x7f) { + p[n++] = v; + break; + } + + p[n++] = (v & 0x7f) | 0x80; + v >>= 7; + } + + ASSERT(n < 6); + + return n; +} + +static inline int tdbGetVarInt(const u8 *p, int *v) { + int n = 0; + int tv = 0; + + for (;;) { + if (p[n] <= 0x7f) { + tv = (tv << 7) | p[n]; + n++; + break; + } + + tv = (tv << 7) | (p[n] & 0x7f); + n++; + } + + ASSERT(n < 6); + + *v = tv; + return n; +} #ifdef __cplusplus } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c new file mode 100644 index 0000000000000000000000000000000000000000..4ec3a895e7dfa99ae33e64701ce560b3a6b288b4 --- /dev/null +++ b/source/libs/tdb/src/page/tdbPage.c @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +extern SPageMethods pageMethods; +extern SPageMethods pageLargeMethods; + +typedef struct __attribute__((__packed__)) { + u16 szCell; + u16 nxOffset; +} SFreeCell; + +static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); +static int tdbPageDefragment(SPage *pPage); + +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { + SPage *pPage; + u8 *ptr; + int size; + + ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); + + *ppPage = NULL; + size = pageSize + sizeof(*pPage); + + ptr = (u8 *)((*xMalloc)(arg, size)); + if (pPage == NULL) { + return -1; + } + + memset(ptr, 0, size); + pPage = (SPage *)(ptr + pageSize); + + pPage->pData = ptr; + pPage->pageSize = pageSize; + if (pageSize < 65536) { + pPage->pPageMethods = &pageMethods; + } else { + pPage->pPageMethods = &pageLargeMethods; + } + TDB_INIT_PAGE_LOCK(pPage); + + /* TODO */ + + *ppPage = pPage; + return 0; +} + +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { + u8 *ptr; + + ptr = pPage->pData; + (*xFree)(arg, ptr); + + return 0; +} + +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + int ret; + SCell *pTarget; + u8 *pTmp; + int j; + + if (pPage->nOverflow || szCell + TDB_PAGE_OFFSET_SIZE(pPage) > pPage->nFree) { + // TODO: need to figure out if pCell may be used by outside of this function + j = pPage->nOverflow++; + + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = idx; + } else { + ret = tdbPageAllocate(pPage, szCell, &pTarget); + if (ret < 0) { + return -1; + } + + memcpy(pTarget, pCell, szCell); + pTmp = pPage->pCellIdx + idx * TDB_PAGE_OFFSET_SIZE(pPage); + memmove(pTmp + TDB_PAGE_OFFSET_SIZE(pPage), pTmp, pPage->pFreeStart - pTmp - TDB_PAGE_OFFSET_SIZE(pPage)); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); + TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); + } + + return 0; +} + +int tdbPageDropCell(SPage *pPage, int idx) { + // TODO + return 0; +} + +static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { + SCell *pCell; + SFreeCell *pFreeCell; + u8 *pOffset; + int ret; + + ASSERT(pPage->nFree > size + TDB_PAGE_OFFSET_SIZE(pPage)); + + pCell = NULL; + *ppCell = NULL; + + // 1. Try to allocate from the free space area + if (pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)) { + pPage->pFreeEnd -= size; + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + pCell = pPage->pFreeEnd; + } + + // 2. Try to allocate from the page free list + if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) && + TDB_PAGE_FCELL(pPage)) { +#if 0 + int szCell; + int nxOffset; + + pCell = pPage->pData + TDB_PAGE_FCELL(pPage); + pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell + : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); + szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); + nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + + for (;;) { + // Find a cell + if (szCell >= size) { + if (szCell - size >= pPage->szFreeCell) { + SCell *pTmpCell = pCell + size; + + TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); + TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); + // TODO: *pOffset = pTmpCell - pPage->pData; + } else { + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); + // TODO: *pOffset = nxOffset; + } + break; + } + + // Not find a cell yet + if (nxOffset > 0) { + pCell = pPage->pData + nxOffset; + pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); + szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); + nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + continue; + } else { + pCell = NULL; + break; + } + } + + if (pCell) { + pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; + } +#endif + } + + // 3. Try to dfragment and allocate again + if (pCell == NULL) { + ret = tdbPageDefragment(pPage); + if (ret < 0) { + return -1; + } + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); + ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + + // Allocate from the free space area again + pPage->pFreeEnd -= size; + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + pCell = pPage->pFreeEnd; + } + + ASSERT(pCell != NULL); + + pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); + *ppCell = pCell; + return 0; +} + +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { + // TODO + return 0; +} + +static int tdbPageDefragment(SPage *pPage) { + // TODO + ASSERT(0); + return 0; +} + +/* ---------------------------------------------------------------------------------------------------------- */ +typedef struct __attribute__((__packed__)) { + u16 flags; + u16 cellNum; + u16 cellBody; + u16 cellFree; + u16 nFree; +} SPageHdr; + +// flags +static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } +static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } + +// cellNum +static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } +static inline void setPageCellNum(SPage *pPage, int cellNum) { + ASSERT(cellNum < 65536); + ((SPageHdr *)(pPage->pPageHdr))[0].cellNum = (u16)cellNum; +} + +// cellBody +static inline int getPageCellBody(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellBody; } +static inline void setPageCellBody(SPage *pPage, int cellBody) { + ASSERT(cellBody < 65536); + ((SPageHdr *)(pPage->pPageHdr))[0].cellBody = (u16)cellBody; +} + +// cellFree +static inline int getPageCellFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellFree; } +static inline void setPageCellFree(SPage *pPage, int cellFree) { + ASSERT(cellFree < 65536); + ((SPageHdr *)(pPage->pPageHdr))[0].cellFree = (u16)cellFree; +} + +// nFree +static inline int getPageNFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].nFree; } +static inline void setPageNFree(SPage *pPage, int nFree) { + ASSERT(nFree < 65536); + ((SPageHdr *)(pPage->pPageHdr))[0].nFree = (u16)nFree; +} + +// cell offset +static inline int getPageCellOffset(SPage *pPage, int idx) { + ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + return ((u16 *)pPage->pCellIdx)[idx]; +} + +static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { + ASSERT(offset < 65536); + ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; +} + +SPageMethods pageMethods = { + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset // setCellOffset +}; \ No newline at end of file diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c new file mode 100644 index 0000000000000000000000000000000000000000..e7c60118d2e279ff9b69608b12883f267a9ad5e6 --- /dev/null +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +typedef struct __attribute__((__packed__)) { + u16 flags; + u8 cellNum[3]; + u8 cellBody[3]; + u8 cellFree[3]; + u8 nFree[3]; +} SPageHdrL; + +typedef struct __attribute__((__packed__)) { + u8 szCell[3]; + u8 nxOffset[3]; +} SFreeCellL; + +// flags +static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } +static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } + +// cellNum +static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } +static inline void setPageCellNum(SPage *pPage, int cellNum) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum, cellNum); +} + +// cellBody +static inline int getPageCellBody(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody); } +static inline void setPageCellBody(SPage *pPage, int cellBody) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody, cellBody); +} + +// cellFree +static inline int getPageCellFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree); } +static inline void setPageCellFree(SPage *pPage, int cellFree) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree, cellFree); +} + +// nFree +static inline int getPageNFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree); } +static inline void setPageNFree(SPage *pPage, int nFree) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree, nFree); +} + +// cell offset +static inline int getPageCellOffset(SPage *pPage, int idx) { + ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + return TDB_GET_U24(pPage->pCellIdx + 3 * idx); +} + +static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { + TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); +} + +SPageMethods pageLargeMethods = { + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset // setCellOffset +}; \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index ad550c7804a280a47835fc91915ebf825a32fce8..c3cc922f32dd5a1f178d59939e66692024b7df1c 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,68 +1,39 @@ #include -#include "tdb.h" +#include "tdbInt.h" TEST(tdb_test, simple_test) { - TENV * pEnv; - TDB * pDb1, *pDb2, *pDb3; - pgsz_t pgSize = 1024; - cachesz_t cacheSize = 10240; - - // ENV - GTEST_ASSERT_EQ(tdbEnvCreate(&pEnv, "./testtdb"), 0); - - GTEST_ASSERT_EQ(tdbEnvSetCache(pEnv, pgSize, cacheSize), 0); - - GTEST_ASSERT_EQ(tdbEnvGetCacheSize(pEnv), cacheSize); - - GTEST_ASSERT_EQ(tdbEnvGetPageSize(pEnv), pgSize); - - GTEST_ASSERT_EQ(tdbEnvOpen(pEnv), 0); - -#if 1 - // DB - GTEST_ASSERT_EQ(tdbCreate(&pDb1), 0); - - // GTEST_ASSERT_EQ(tdbSetKeyLen(pDb1, 8), 0); - - // GTEST_ASSERT_EQ(tdbGetKeyLen(pDb1), 8); - - // GTEST_ASSERT_EQ(tdbSetValLen(pDb1, 3), 0); - - // GTEST_ASSERT_EQ(tdbGetValLen(pDb1), 3); - - // GTEST_ASSERT_EQ(tdbSetDup(pDb1, 1), 0); - - // GTEST_ASSERT_EQ(tdbGetDup(pDb1), 1); - - // GTEST_ASSERT_EQ(tdbSetCmprFunc(pDb1, NULL), 0); - - tdbEnvBeginTxn(pEnv); - - GTEST_ASSERT_EQ(tdbOpen(pDb1, "db.db", "db1", pEnv), 0); - - // char *key = "key1"; - // char *val = "value1"; - // tdbInsert(pDb1, (void *)key, strlen(key), (void *)val, strlen(val)); - - tdbEnvCommit(pEnv); - -#if 0 - // Insert - - // Query - - // Delete - - // Query -#endif - - // GTEST_ASSERT_EQ(tdbOpen(&pDb2, "db.db", "db2", pEnv), 0); - // GTEST_ASSERT_EQ(tdbOpen(&pDb3, "index.db", NULL, pEnv), 0); - // tdbClose(pDb3); - // tdbClose(pDb2); - tdbClose(pDb1); -#endif - - tdbEnvClose(pEnv); + int ret; + STEnv *pEnv; + STDb *pDb; + + // Open Env + ret = tdbEnvOpen("tdb", 1024, 20, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + { // Insert some data + char key[64]; + char val[64]; + + for (int i = 1; i <= 1000; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); + GTEST_ASSERT_EQ(ret, 0); + } + } + + ret = tdbDbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbDbClose(pDb); + + // Close Env + ret = tdbEnvClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } \ No newline at end of file diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index f16d0445c6bb2ab9eb5aaf6e5d7844527637b538..2a508cf676f744d0ffd09e4b656fcb15df63c82a 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -41,7 +41,7 @@ typedef struct { } STfsDisk; typedef struct { - pthread_spinlock_t lock; + TdThreadSpinlock lock; int32_t level; int32_t nextid; // next disk id to allocate int32_t ndisk; // # of disks mounted to this tier @@ -64,7 +64,7 @@ typedef struct STfsDir { } STfsDir; typedef struct STfs { - pthread_spinlock_t lock; + TdThreadSpinlock lock; SDiskSize size; int32_t nlevel; STfsTier tiers[TFS_MAX_TIERS]; @@ -82,11 +82,11 @@ void tfsUpdateTierSize(STfsTier *pTier); int32_t tfsAllocDiskOnTier(STfsTier *pTier); void tfsPosNextId(STfsTier *pTier); -#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock) -#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) +#define tfsLockTier(pTier) taosThreadSpinLock(&(pTier)->lock) +#define tfsUnLockTier(pTier) taosThreadSpinUnlock(&(pTier)->lock) -#define tfsLock(pTfs) pthread_spin_lock(&(pTfs)->lock) -#define tfsUnLock(pTfs) pthread_spin_unlock(&(pTfs)->lock) +#define tfsLock(pTfs) taosThreadSpinLock(&(pTfs)->lock) +#define tfsUnLock(pTfs) taosThreadSpinUnlock(&(pTfs)->lock) #define TFS_TIER_AT(pTfs, level) (&(pTfs)->tiers[level]) #define TFS_DISK_AT(pTfs, did) ((pTfs)->tiers[(did).level].disks[(did).id]) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index c46989dc5dbeda6e5b774cc817ed7a97713f8087..bfe7904a63e3c64b9c1e0c9a19c8ff784c9998c0 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -36,7 +36,7 @@ STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { return NULL; } - if (pthread_spin_init(&pTfs->lock, 0) != 0) { + if (taosThreadSpinInit(&pTfs->lock, 0) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); tfsClose(pTfs); return NULL; @@ -85,7 +85,7 @@ void tfsClose(STfs *pTfs) { } taosHashCleanup(pTfs->hash); - pthread_spin_destroy(&pTfs->lock); + taosThreadSpinDestroy(&pTfs->lock); free(pTfs); } @@ -202,6 +202,12 @@ void tfsDirname(const STfsFile *pFile, char *dest) { tstrncpy(dest, taosDirName(tname), TSDB_FILENAME_LEN); } +void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname) { + STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); + + snprintf(aname, TSDB_FILENAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); +} + int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); } int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) { @@ -389,7 +395,7 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { } static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { - char dirName[TSDB_FILENAME_LEN] = "\0"; + char dirName[TSDB_FILENAME_LEN] = "\0"; if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); @@ -558,4 +564,6 @@ int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo) { } } tfsUnLock(pTfs); -} \ No newline at end of file + + return 0; +} diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index e4390d13d11a8b331d4e13a93e10f0ec5c1397c2..2ad021cbb543ff3f65ec9a558278451c69791d31 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -19,7 +19,7 @@ int32_t tfsInitTier(STfsTier *pTier, int32_t level) { memset(pTier, 0, sizeof(STfsTier)); - if (pthread_spin_init(&pTier->lock, 0) != 0) { + if (taosThreadSpinInit(&pTier->lock, 0) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -34,7 +34,7 @@ void tfsDestroyTier(STfsTier *pTier) { } pTier->ndisk = 0; - pthread_spin_destroy(&pTier->lock); + taosThreadSpinDestroy(&pTier->lock); } STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) { diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 5cc436cf3267eb68bf106f018512ea661512c92f..20dd3f7ad2865305c3ad663fbc29e948f9624895 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common - PUBLIC zlib + PUBLIC zlibstatic ) if (${BUILD_WITH_UV_TRANS}) if (${BUILD_WITH_UV}) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 99f890d3a0288b6da16c8047267986213a43e5fd..d4e75dcd843aed23ca4cedbc835ef1af532442b3 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -14,6 +14,10 @@ */ #ifdef USE_UV +#ifdef __cplusplus +extern "C" { +#endif + #include #include "lz4.h" #include "os.h" @@ -121,25 +125,21 @@ typedef struct { } SRpcReqContext; typedef SRpcMsg STransMsg; +typedef SRpcCtx STransCtx; +typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; typedef struct { - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - // struct SRpcConn* pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app - int32_t contLen; // content length - // int32_t code; // error code - // int16_t numOfTry; // number of try for different servers - // int8_t oldInUse; // server EP inUse passed by app - // int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + tmsg_t msgType; // message type + int8_t connType; // connection type cli/srv int64_t rid; // refId returned by taosAddRef - STransMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API + STransCtx appCtx; // + STransMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API int hThrdIdx; char* ip; @@ -151,11 +151,12 @@ typedef struct { typedef struct { char version : 4; // RPC version - char comp : 4; // compression algorithm, 0:no compression 1:lz4 - char resflag : 2; // reserved bits - char spi : 1; // security parameter index + char comp : 2; // compression algorithm, 0:no compression 1:lz4 + char noResp : 2; // noResp bits, 0: resp, 1: resp + char persist : 2; // persist handle,0: no persit, 1: persist handle + char release : 2; char secured : 2; - char encrypt : 3; // encrypt algorithm, 0: no encryption + char spi : 2; uint32_t code; // del later uint32_t msgType; @@ -180,6 +181,9 @@ typedef struct { #pragma pack(pop) +typedef enum { Normal, Quit, Release, Register } STransMsgType; +typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken } ConnStatus; + #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) #define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) @@ -226,7 +230,7 @@ typedef void (*AsyncCB)(uv_async_t* handle); typedef struct { void* pThrd; queue qmsg; - pthread_mutex_t mtx; // protect qmsg; + TdThreadMutex mtx; // protect qmsg; } SAsyncItem; typedef struct { @@ -253,9 +257,13 @@ void transUnrefSrvHandle(void* handle); void transRefCliHandle(void* handle); void transUnrefCliHandle(void* handle); -void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg); +void transReleaseCliHandle(void* handle); +void transReleaseSrvHandle(void* handle); + +void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* pCtx); void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp); -void transSendResponse(const STransMsg* pMsg); +void transSendResponse(const STransMsg* msg); +void transRegisterMsg(const STransMsg* msg); int transGetConnInfo(void* thandle, STransHandleInfo* pInfo); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); @@ -264,4 +272,14 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void transCloseClient(void* arg); void transCloseServer(void* arg); +void transCtxInit(STransCtx* ctx); +void transCtxDestroy(STransCtx* ctx); +void transCtxClear(STransCtx* ctx); +void transCtxMerge(STransCtx* dst, STransCtx* src); +void* transCtxDumpVal(STransCtx* ctx, int32_t key); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 3924a5cf1a022349233eef43e57a00ff5d56bb40..ad50948a021ac663bcc012ff6cbf08f86a354e4e 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -63,8 +63,6 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); - bool (*pfp)(void* parent, tmsg_t msgType); - void* (*mfp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; @@ -72,7 +70,7 @@ typedef struct { void* tmrCtrl; // handle to timer SHashObj* hash; // handle returned by hash utility void* tcphandle; // returned handle from TCP initialization - pthread_mutex_t mutex; + TdThreadMutex mutex; } SRpcInfo; #endif // USE_LIBUV diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 1db2808126a162e4b15305ff80057955de1025a3..d629f6dba4b958137756bbb867df4e9c23018aa4 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -39,7 +39,7 @@ typedef struct { int total; int * count; int64_t keepTimer; - pthread_mutex_t mutex; + TdThreadMutex mutex; void (*cleanFp)(void *); void * tmrCtrl; void * pTimer; @@ -85,7 +85,7 @@ void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, pCache->lockedBy = calloc(sizeof(int64_t), maxSessions); taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); - pthread_mutex_init(&pCache->mutex, NULL); + taosThreadMutexInit(&pCache->mutex, NULL); return pCache; } @@ -96,7 +96,7 @@ void rpcCloseConnCache(void *handle) { pCache = (SConnCache *)handle; if (pCache == NULL || pCache->maxSessions == 0) return; - pthread_mutex_lock(&pCache->mutex); + taosThreadMutexLock(&pCache->mutex); taosTmrStopA(&(pCache->pTimer)); @@ -106,9 +106,9 @@ void rpcCloseConnCache(void *handle) { tfree(pCache->count); tfree(pCache->lockedBy); - pthread_mutex_unlock(&pCache->mutex); + taosThreadMutexUnlock(&pCache->mutex); - pthread_mutex_destroy(&pCache->mutex); + taosThreadMutexDestroy(&pCache->mutex); memset(pCache, 0, sizeof(SConnCache)); free(pCache); @@ -220,7 +220,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { if (pCache == NULL || pCache->maxSessions == 0) return; if (pCache->pTimer != tmrId) return; - pthread_mutex_lock(&pCache->mutex); + taosThreadMutexLock(&pCache->mutex); uint64_t time = taosGetTimestampMs(); for (hash = 0; hash < pCache->maxSessions; ++hash) { @@ -232,7 +232,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { // tTrace("timer, total connections in cache:%d", pCache->total); taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); - pthread_mutex_unlock(&pCache->mutex); + taosThreadMutexUnlock(&pCache->mutex); } static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) { diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 86dd17bb0c8978c04d3d031f827b1b23d5a78d43..98b24ac3c7cb4e6404b94b8780c2fdbcb2d16f2f 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -14,6 +14,7 @@ */ #include "lz4.h" +#include "transportInt.h" #include "os.h" #include "rpcCache.h" #include "rpcHead.h" @@ -27,13 +28,12 @@ #include "tmd5.h" #include "tmempool.h" #include "tmsg.h" -#include "transportInt.h" #include "tref.h" #include "trpc.h" #include "ttimer.h" #include "tutil.h" -static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; +static TdThreadOnce tsRpcInitOnce = PTHREAD_ONCE_INIT; int tsRpcMaxUdpSize = 15000; // bytes int tsProgressTimer = 100; @@ -72,7 +72,7 @@ typedef struct { void * tcphandle; // returned handle from TCP initialization void * udphandle; // returned handle from UDP initialization void * pCache; // connection cache - pthread_mutex_t mutex; + TdThreadMutex mutex; struct SRpcConn *connList; // connection list } SRpcInfo; @@ -143,7 +143,7 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; -// static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; +// static TdThreadOnce tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 #define RPC_CONN_UDPS 0 @@ -223,7 +223,7 @@ static void rpcInitImp(void) { } int32_t rpcInit() { - pthread_once(&tsRpcInitOnce, rpcInitImp); + taosThreadOnce(&tsRpcInitOnce, rpcInitImp); return 0; } @@ -238,7 +238,7 @@ void rpcCleanup(void) { void *rpcOpen(const SRpcInit *pInit) { SRpcInfo *pRpc; - // pthread_once(&tsRpcInit, rpcInit); + // taosThreadOnce(&tsRpcInit, rpcInit); pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; @@ -307,7 +307,7 @@ void *rpcOpen(const SRpcInit *pInit) { } } - pthread_mutex_init(&pRpc->mutex, NULL); + taosThreadMutexInit(&pRpc->mutex, NULL); pRpc->tcphandle = (*taosInitConn[pRpc->connType | RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); @@ -1672,7 +1672,7 @@ static void rpcDecRef(SRpcInfo *pRpc) { taosIdPoolCleanUp(pRpc->idPool); tfree(pRpc->connList); - pthread_mutex_destroy(&pRpc->mutex); + taosThreadMutexDestroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); tfree(pRpc); diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index aac38b21e8fdf570dfe76414586e6cca83961dd7..4b1700a10f882c64fa4f74cda67634b782ba3421 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -35,9 +35,9 @@ typedef struct SFdObj { } SFdObj; typedef struct SThreadObj { - pthread_t thread; + TdThread thread; SFdObj * pHead; - pthread_mutex_t mutex; + TdThreadMutex mutex; uint32_t ip; bool stop; TdEpollPtr pEpoll; @@ -65,7 +65,7 @@ typedef struct { int numOfThreads; void * shandle; SThreadObj **pThreadObj; - pthread_t thread; + TdThread thread; } SServerObj; static void * taosProcessTcpData(void *param); @@ -101,9 +101,9 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } int code = 0; - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); // initialize parameters in case it may encounter error later for (int i = 0; i < numOfThreads; ++i) { @@ -129,7 +129,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread // initialize mutex, thread, fd which may fail for (int i = 0; i < numOfThreads; ++i) { pThreadObj = pServerObj->pThreadObj[i]; - code = pthread_mutex_init(&(pThreadObj->mutex), NULL); + code = taosThreadMutexInit(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break; @@ -142,7 +142,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + code = taosThreadCreate(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); break; @@ -155,7 +155,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread if (pServerObj->pSocketServer == NULL) code = -1; if (code == 0) { - code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); + code = taosThreadCreate(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); if (code != 0) { tError("%s failed to create TCP accept thread(%s)", label, strerror(code)); } @@ -169,7 +169,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread tDebug("%s TCP server is initialized, ip:0x%x port:%hu numOfThreads:%d", label, ip, port, numOfThreads); } - pthread_attr_destroy(&thattr); + taosThreadAttrDestroy(&thattr); return (void *)pServerObj; } @@ -178,16 +178,16 @@ static void taosStopTcpThread(SThreadObj *pThreadObj) { return; } // save thread into local variable and signal thread to stop - pthread_t thread = pThreadObj->thread; + TdThread thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { return; } pThreadObj->stop = true; - if (taosComparePthread(thread, pthread_self())) { - pthread_detach(pthread_self()); + if (taosComparePthread(thread, taosThreadSelf())) { + pthread_detach(taosThreadSelf()); return; } - pthread_join(thread, NULL); + taosThreadJoin(thread, NULL); } void taosStopTcpServer(void *handle) { @@ -200,10 +200,10 @@ void taosStopTcpServer(void *handle) { taosShutDownSocketServerRD(pServerObj->pSocketServer); } if (taosCheckPthreadValid(pServerObj->thread)) { - if (taosComparePthread(pServerObj->thread, pthread_self())) { - pthread_detach(pthread_self()); + if (taosComparePthread(pServerObj->thread, taosThreadSelf())) { + pthread_detach(taosThreadSelf()); } else { - pthread_join(pServerObj->thread, NULL); + taosThreadJoin(pServerObj->thread, NULL); } } @@ -307,9 +307,9 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread } int code = 0; - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < numOfThreads; ++i) { SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj)); @@ -318,7 +318,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread terrno = TAOS_SYSTEM_ERROR(errno); for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]); free(pClientObj); - pthread_attr_destroy(&thattr); + taosThreadAttrDestroy(&thattr); return NULL; } pClientObj->pThreadObj[i] = pThreadObj; @@ -333,7 +333,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread // initialize mutex, thread, fd which may fail for (int i = 0; i < numOfThreads; ++i) { SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; - code = pthread_mutex_init(&(pThreadObj->mutex), NULL); + code = taosThreadMutexInit(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break; @@ -346,7 +346,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + code = taosThreadCreate(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); break; @@ -578,7 +578,7 @@ static void *taosProcessTcpData(void *param) { taosReportBrokenLink(pFdObj); } - pthread_mutex_destroy(&(pThreadObj->mutex)); + taosThreadMutexDestroy(&(pThreadObj->mutex)); tDebug("%s TCP thread exits ...", pThreadObj->label); tfree(pThreadObj); @@ -607,12 +607,12 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { } // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->mutex)); + taosThreadMutexLock(&(pThreadObj->mutex)); pFdObj->next = pThreadObj->pHead; if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; pThreadObj->pHead = pFdObj; pThreadObj->numOfFds++; - pthread_mutex_unlock(&(pThreadObj->mutex)); + taosThreadMutexUnlock(&(pThreadObj->mutex)); return pFdObj; } @@ -622,10 +622,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) { if (pFdObj->signature != pFdObj) return; SThreadObj *pThreadObj = pFdObj->pThreadObj; - pthread_mutex_lock(&pThreadObj->mutex); + taosThreadMutexLock(&pThreadObj->mutex); if (pFdObj->signature == NULL) { - pthread_mutex_unlock(&pThreadObj->mutex); + taosThreadMutexUnlock(&pThreadObj->mutex); return; } @@ -648,7 +648,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { (pFdObj->next)->prev = pFdObj->prev; } - pthread_mutex_unlock(&pThreadObj->mutex); + taosThreadMutexUnlock(&pThreadObj->mutex); tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 81c8d0af7616598c886ce9dad9fb5be16c83c117..af26761603f73082ad7dabffb6d7961aee6f3c0c 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -35,7 +35,7 @@ typedef struct { uint16_t port; // peer port uint16_t localPort; // local port char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - pthread_t thread; + TdThread thread; void *hash; void *shandle; // handle passed by upper layer during server initialization void *pSet; @@ -77,9 +77,9 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pSet->threads = threads; tstrncpy(pSet->label, label, sizeof(pSet->label)); - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); int i; uint16_t ownPort; @@ -111,14 +111,14 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pConn->index = i; pConn->pSet = pSet; - int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); + int code = taosThreadCreate(&pConn->thread, &thAttr, taosRecvUdpData, pConn); if (code != 0) { tError("%s failed to create thread to process UDP data(%s)", label, strerror(errno)); break; } } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); if (i != threads) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -146,7 +146,7 @@ void taosStopUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; if (taosCheckPthreadValid(pConn->thread)) { - pthread_join(pConn->thread, NULL); + taosThreadJoin(pConn->thread, NULL); } tfree(pConn->buffer); // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6d1c691b9ba17ac001bfcdf35212cc225f11b0d6..4e2cb835279b7b9ae951b8094704bd5c98577fbd 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -14,10 +14,13 @@ */ #define _DEFAULT_SOURCE +#ifdef USE_UV +#include +#endif +#include "zlib.h" #include "thttp.h" #include "taoserror.h" #include "tlog.h" -#include "zlib.h" static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pHead, int32_t headLen, EHttpCompFlag flag) { @@ -111,7 +114,6 @@ _OVER: } #ifdef USE_UV -#include static void clientConnCb(uv_connect_t* req, int32_t status) { if (status < 0) { terrno = TAOS_SYSTEM_ERROR(status); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 58809ee3be24519d494bd2161cd44859f3417312..9d0fba488510a5c5492fda3795b007760b5d0936 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -22,6 +22,11 @@ void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThre void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; +void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; +void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle}; + +void (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; + void* rpcOpen(const SRpcInit* pInit) { SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) { @@ -34,8 +39,6 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; - pRpc->pfp = pInit->pfp; - pRpc->mfp = pInit->mfp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; @@ -115,7 +118,12 @@ void rpcCancelRequest(int64_t rid) { return; } void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); uint32_t port = pEpSet->eps[pEpSet->inUse].port; - transSendRequest(shandle, ip, port, pMsg); + transSendRequest(shandle, ip, port, pMsg, NULL); +} +void rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { + char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); + uint32_t port = pEpSet->eps[pEpSet->inUse].port; + transSendRequest(shandle, ip, port, pMsg, pCtx); } void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); @@ -126,9 +134,6 @@ void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { void rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); } int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return transGetConnInfo((void*)thandle, pInfo); } -void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; -void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle}; - void rpcRefHandle(void* handle, int8_t type) { assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); (*taosRefHandle[type])(handle); @@ -139,6 +144,12 @@ void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } +void rpcRegisterBrokenLinkArg(SRpcMsg* msg) { transRegisterMsg(msg); } +void rpcReleaseHandle(void* handle, int8_t type) { + assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); + (*transReleaseHandle[type])(handle); +} + int32_t rpcInit() { // impl later return 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2f6ff3763f9255f8613cb6ca2cdfd4c4db729800..fe5d8bd7f5de8ac85a8c4f34c98e3f0d2b3efb8c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -25,15 +25,22 @@ typedef struct SCliConn { void* hostThrd; SConnBuffer readBuf; void* data; + SArray* cliMsgs; queue conn; uint64_t expireTime; int hThrdIdx; bool broken; // link broken or not + STransCtx ctx; - int persist; // + ConnStatus status; // + int release; // 1: release // spi configure char spi; char secured; + + char* ip; + uint32_t port; + // debug and log info struct sockaddr_in addr; struct sockaddr_in locaddr; @@ -45,10 +52,11 @@ typedef struct SCliMsg { STransMsg msg; queue q; uint64_t st; + STransMsgType type; } SCliMsg; typedef struct SCliThrdObj { - pthread_t thread; + TdThread thread; uv_loop_t* loop; SAsyncPool* asyncPool; uv_timer_t timer; @@ -56,7 +64,7 @@ typedef struct SCliThrdObj { // msg queue queue msg; - pthread_mutex_t msgMtx; + TdThreadMutex msgMtx; uint64_t nextTimeout; // next timeout void* pTransInst; // @@ -79,7 +87,7 @@ typedef struct SConnList { static void* createConnPool(int size); static void* destroyConnPool(void* pool); static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port); -static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn); +static void addConnToPool(void* pool, SCliConn* conn); // register timer in each thread to clear expire conn static void cliTimeoutCb(uv_timer_t* handle); @@ -96,14 +104,19 @@ static void cliAsyncCb(uv_async_t* handle); static SCliConn* cliCreateConn(SCliThrdObj* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void cliDestroy(uv_handle_t* handle); +static void cliSend(SCliConn* pConn); // process data read from server, add decompress etc later static void cliHandleResp(SCliConn* conn); // handle except about conn static void cliHandleExcept(SCliConn* conn); + // handle req from app static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease}; + static void cliSendQuit(SCliThrdObj* thrd); static void destroyUserdata(STransMsg* userdata); @@ -117,35 +130,64 @@ static void destroyThrdObj(SCliThrdObj* pThrd); #define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) +#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tDebug("cli conn %p receive release request, ref: %d", conn, T_REF_VAL_GET(conn)); \ + while (T_REF_VAL_GET(conn) > 1) { \ + transUnrefCliHandle(conn); \ + } \ + if (T_REF_VAL_GET(conn) == 1) { \ + SCliThrdObj* thrd = conn->hostThrd; \ + addConnToPool(thrd->pool, conn); \ + } \ + return; \ + } \ + } while (0) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)conn->hostThrd)->pTransInst))->label) -#define CONN_HANDLE_THREAD_QUIT(conn, thrd) \ - do { \ - if (thrd->quit) { \ - cliHandleExcept(conn); \ - goto _RETURE; \ - } \ +#define CONN_HANDLE_THREAD_QUIT(thrd) \ + do { \ + if (thrd->quit) { \ + return; \ + } \ } while (0) #define CONN_HANDLE_BROKEN(conn) \ do { \ if (conn->broken) { \ cliHandleExcept(conn); \ - goto _RETURE; \ + return; \ } \ - } while (0); + } while (0) #define CONN_SET_PERSIST_BY_APP(conn) \ do { \ - if (conn->persist == false) { \ - conn->persist = true; \ + if (conn->status == ConnNormal) { \ + conn->status = ConnAcquire; \ transRefCliHandle(conn); \ } \ } while (0) -#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false) +#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->status == ConnNormal && T_REF_VAL_GET(conn) == 1) + +#define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) +#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) +#define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) static void* cliWorkThread(void* arg); +bool cliMaySendCachedMsg(SCliConn* conn) { + if (taosArrayGetSize(conn->cliMsgs) > 0) { + cliSend(conn); + return true; + } else { + return false; + } +} void cliHandleResp(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; STrans* pTransInst = pThrd->pTransInst; @@ -153,7 +195,6 @@ void cliHandleResp(SCliConn* conn) { STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); - STransMsg transMsg = {0}; transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = transContFromHead((char*)pHead); @@ -161,33 +202,40 @@ void cliHandleResp(SCliConn* conn) { transMsg.msgType = pHead->msgType; transMsg.ahandle = NULL; - SCliMsg* pMsg = conn->data; + CONN_SHOULD_RELEASE(conn, pHead); + + SCliMsg* pMsg = NULL; + if (taosArrayGetSize(conn->cliMsgs) > 0) { + pMsg = taosArrayGetP(conn->cliMsgs, 0); + taosArrayRemove(conn->cliMsgs, 0); + } + STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; + transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); } else { transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } - // if (rpcMsg.ahandle == NULL) { - // tDebug("%s cli conn %p handle except", CONN_GET_INST_LABEL(conn), conn); - // return; - //} - // buf's mem alread translated to transMsg.pCont transClearBuffer(&conn->readBuf); - if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, transMsg.msgType)) { + if (!CONN_NO_PERSIST_BY_APP(conn)) { transMsg.handle = conn; - CONN_SET_PERSIST_BY_APP(conn); tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, - TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), - inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); + TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), + taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); conn->secured = pHead->secured; + if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { + tTrace("except, server continue send while cli ignore it"); + // transUnrefCliHandle(conn); + return; + } + if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, conn); (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); @@ -196,23 +244,27 @@ void cliHandleResp(SCliConn* conn) { memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg)); tsem_post(pCtx->pSem); } + destroyCmsg(pMsg); - uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb); + if (cliMaySendCachedMsg(conn) == true) { + return; + } if (CONN_NO_PERSIST_BY_APP(conn)) { - addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + addConnToPool(pThrd->pool, conn); } - destroyCmsg(conn->data); - conn->data = NULL; + uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb); // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)&pThrd->timer) && pTransInst->idleTime > 0) { // uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } +_RETURN: + return; } void cliHandleExcept(SCliConn* pConn) { - if (pConn->data == NULL) { + if (taosArrayGetSize(pConn->cliMsgs) == 0) { if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) { transUnrefCliHandle(pConn); return; @@ -221,32 +273,38 @@ void cliHandleExcept(SCliConn* pConn) { SCliThrdObj* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; - SCliMsg* pMsg = pConn->data; - STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; + do { + SCliMsg* pMsg = NULL; + if (taosArrayGetSize(pConn->cliMsgs) > 0) { + pMsg = taosArrayGetP(pConn->cliMsgs, 0); + taosArrayRemove(pConn->cliMsgs, 0); + } - STransMsg transMsg = {0}; - transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; - transMsg.ahandle = NULL; + STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; - if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { - transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; - } else { - transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; - } + STransMsg transMsg = {0}; + transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; + transMsg.ahandle = NULL; - if (pCtx == NULL || pCtx->pSem == NULL) { - tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); - (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); - } else { - tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, pConn); - memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); - tsem_post(pCtx->pSem); - } - destroyCmsg(pConn->data); - pConn->data = NULL; + if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { + transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); + } else { + transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + } + + if (pCtx == NULL || pCtx->pSem == NULL) { + tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); + (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + } else { + tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, pConn); + memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); + tsem_post(pCtx->pSem); + } + destroyCmsg(pMsg); + tTrace("%s cli conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); + } while (taosArrayGetSize(pConn->cliMsgs) > 0); - tTrace("%s cli conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); transUnrefCliHandle(pConn); } @@ -291,6 +349,7 @@ void* destroyConnPool(void* pool) { connList = taosHashIterate((SHashObj*)pool, connList); } taosHashCleanup(pool); + return NULL; } static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { @@ -317,20 +376,27 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { QUEUE_INIT(&conn->conn); return conn; } -static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { +static void addConnToPool(void* pool, SCliConn* conn) { + SCliThrdObj* thrd = conn->hostThrd; + CONN_HANDLE_THREAD_QUIT(thrd); + char key[128] = {0}; - tstrncpy(key, ip, strlen(ip)); - tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); + transCtxDestroy(&conn->ctx); + tstrncpy(key, conn->ip, strlen(conn->ip)); + tstrncpy(key + strlen(key), (char*)(&conn->port), sizeof(conn->port)); tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); STrans* pTransInst = ((SCliThrdObj*)conn->hostThrd)->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); + conn->status = ConnNormal; // list already create before assert(plist != NULL); + taosArrayClear(conn->cliMsgs); QUEUE_PUSH(&plist->conn, &conn->conn); + assert(!QUEUE_IS_EMPTY(&plist->conn)); } static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; @@ -360,6 +426,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb // nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under // read(2). + tTrace("%s cli conn %p read empty", CONN_GET_INST_LABEL(conn), conn); return; } if (nread < 0) { @@ -378,16 +445,17 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { conn->writeReq.data = conn; conn->connReq.data = conn; - + conn->cliMsgs = taosArrayInit(2, sizeof(void*)); QUEUE_INIT(&conn->conn); conn->hostThrd = pThrd; - conn->persist = false; - conn->broken = false; + conn->status = ConnNormal; + conn->broken = 0; transRefCliHandle(conn); return conn; } static void cliDestroyConn(SCliConn* conn, bool clear) { tTrace("%s cli conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); + QUEUE_REMOVE(&conn->conn); if (clear) { uv_close((uv_handle_t*)conn->stream, cliDestroy); @@ -395,41 +463,64 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } static void cliDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; - + free(conn->ip); free(conn->stream); + transCtxDestroy(&conn->ctx); + taosArrayDestroy(conn->cliMsgs); tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); free(conn); } - +static bool cliHandleNoResp(SCliConn* conn) { + bool res = false; + SArray* msgs = conn->cliMsgs; + if (taosArrayGetSize(msgs) > 0) { + SCliMsg* pMsg = taosArrayGetP(msgs, 0); + if (REQUEST_NO_RESP(&pMsg->msg)) { + taosArrayRemove(msgs, 0); + destroyCmsg(pMsg); + res = true; + } + if (res == true) { + if (cliMaySendCachedMsg(conn) == false) { + SCliThrdObj* thrd = conn->hostThrd; + addConnToPool(thrd->pool, conn); + } + } + } + return res; +} static void cliSendCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; if (status == 0) { tTrace("%s cli conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); - SCliMsg* pMsg = pConn->data; - if (pMsg == NULL) { - return; - } - destroyUserdata(&pMsg->msg); } else { tError("%s cli conn %p failed to write: %s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status)); cliHandleExcept(pConn); return; } + if (cliHandleNoResp(pConn) == true) { + tTrace("%s cli conn %p no resp required", CONN_GET_INST_LABEL(pConn), pConn); + return; + } uv_read_start((uv_stream_t*)pConn->stream, cliAllocRecvBufferCb, cliRecvCb); } void cliSend(SCliConn* pConn) { CONN_HANDLE_BROKEN(pConn); - SCliMsg* pCliMsg = pConn->data; + assert(taosArrayGetSize(pConn->cliMsgs) > 0); + SCliMsg* pCliMsg = taosArrayGetP(pConn->cliMsgs, 0); STransConnCtx* pCtx = pCliMsg->ctx; SCliThrdObj* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); - + if (pMsg->pCont == 0) { + pMsg->pCont = (void*)rpcMallocCont(0); + pMsg->contLen = 0; + } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); int msgLen = transMsgLenFromCont(pMsg->contLen); @@ -452,18 +543,24 @@ void cliSend(SCliConn* pConn) { msgLen += sizeof(STransUserMsg); } + pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; + pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; pHead->msgType = pMsg->msgType; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, - TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + + if (pHead->persist == 1) { + CONN_SET_PERSIST_BY_APP(pConn); + } + pConn->writeReq.data = pConn; uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); - return; -_RETURE: return; } @@ -482,8 +579,8 @@ void cliConnCb(uv_connect_t* req, int status) { uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); tTrace("%s cli conn %p connect to server successfully", CONN_GET_INST_LABEL(pConn), pConn); - assert(pConn->stream == req->handle); + cliSend(pConn); } @@ -497,6 +594,22 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { pThrd->quit = true; uv_stop(pThrd->loop); } +static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { + SCliConn* conn = pMsg->msg.handle; + tDebug("%s cli conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); + + if (T_REF_VAL_GET(conn) == 2) { + transUnrefCliHandle(conn); + taosArrayPush(conn->cliMsgs, &pMsg); + if (taosArrayGetSize(conn->cliMsgs) >= 2) { + return; // send one by one + } + cliSend(conn); + } else { + // conn already broken down + transUnrefCliHandle(conn); + } +} SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = NULL; @@ -508,7 +621,11 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { } else { STransConnCtx* pCtx = pMsg->ctx; conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); - if (conn != NULL) tTrace("%s cli conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + if (conn != NULL) { + tTrace("%s cli conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + } else { + tTrace("not found conn in conn pool %p", pThrd->pool); + } } return conn; } @@ -523,12 +640,24 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { - conn->data = pMsg; + conn->hThrdIdx = pCtx->hThrdIdx; + + transCtxMerge(&conn->ctx, &pCtx->appCtx); + if (taosArrayGetSize(conn->cliMsgs) > 0) { + taosArrayPush(conn->cliMsgs, &pMsg); + return; + } + + taosArrayPush(conn->cliMsgs, &pMsg); transDestroyBuffer(&conn->readBuf); cliSend(conn); } else { conn = cliCreateConn(pThrd); - conn->data = pMsg; + taosArrayPush(conn->cliMsgs, &pMsg); + + conn->hThrdIdx = pCtx->hThrdIdx; + conn->ip = strdup(pMsg->ctx->ip); + conn->port = pMsg->ctx->port; int ret = transSetConnOption((uv_tcp_t*)conn->stream); if (ret) { @@ -540,8 +669,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tTrace("%s cli conn %p try to connect to %s:%d", pTransInst->label, conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); } - - conn->hThrdIdx = pCtx->hThrdIdx; } static void cliAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; @@ -550,9 +677,9 @@ static void cliAsyncCb(uv_async_t* handle) { // batch process to avoid to lock/unlock frequently queue wq; - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_MOVE(&item->qmsg, &wq); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); int count = 0; while (!QUEUE_IS_EMPTY(&wq)) { @@ -560,11 +687,10 @@ static void cliAsyncCb(uv_async_t* handle) { QUEUE_REMOVE(h); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - if (pMsg->ctx == NULL) { - cliHandleQuit(pMsg, pThrd); - } else { - cliHandleReq(pMsg, pThrd); + if (pMsg == NULL) { + continue; } + (*cliAsyncHandle[pMsg->type])(pMsg, pThrd); count++; } if (count >= 2) { @@ -576,6 +702,8 @@ static void* cliWorkThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; setThreadName("trans-cli-work"); uv_run(pThrd->loop, UV_RUN_DEFAULT); + + return NULL; } void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { @@ -591,7 +719,7 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = shandle; - int err = pthread_create(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); + int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); if (err == 0) { tDebug("success to create tranport-cli thread %d", i); } @@ -620,7 +748,7 @@ static SCliThrdObj* createThrdObj() { SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); + taosThreadMutexInit(&pThrd->msgMtx, NULL); pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); @@ -640,8 +768,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { return; } uv_stop(pThrd->loop); - pthread_join(pThrd->thread, NULL); - pthread_mutex_destroy(&pThrd->msgMtx); + taosThreadJoin(pThrd->thread, NULL); + taosThreadMutexDestroy(&pThrd->msgMtx); transDestroyAsyncPool(pThrd->asyncPool); uv_timer_stop(&pThrd->timer); @@ -659,8 +787,10 @@ static void transDestroyConnCtx(STransConnCtx* ctx) { void cliSendQuit(SCliThrdObj* thrd) { // cli can stop gracefully SCliMsg* msg = calloc(1, sizeof(SCliMsg)); + msg->type = Quit; transSendAsync(thrd->asyncPool, &msg->q); } + int cliRBChoseIdx(STrans* pTransInst) { int64_t index = pTransInst->index; if (pTransInst->index++ >= pTransInst->numOfThreads) { @@ -690,22 +820,32 @@ void transUnrefCliHandle(void* handle) { return; } int ref = T_REF_DEC((SCliConn*)handle); + tDebug("%s cli conn %p ref %d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref); if (ref == 0) { cliDestroyConn((SCliConn*)handle, true); } } +void transReleaseCliHandle(void* handle) { + SCliThrdObj* thrd = CONN_GET_HOST_THREAD(handle); + if (thrd == NULL) { + return; + } + + STransMsg tmsg = {.handle = handle}; + SCliMsg* cmsg = calloc(1, sizeof(SCliMsg)); + cmsg->msg = tmsg; + cmsg->type = Release; + + transSendAsync(thrd->asyncPool, &cmsg->q); +} -void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg) { +void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* ctx) { STrans* pTransInst = (STrans*)shandle; int index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle); if (index == -1) { index = cliRBChoseIdx(pTransInst); } - int32_t flen = 0; - if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) { - // imp later - } - tDebug("send request at thread:%d %p", index, pMsg); + STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; @@ -713,17 +853,23 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p pCtx->port = port; pCtx->hThrdIdx = index; + if (ctx != NULL) { + pCtx->appCtx = *ctx; + } assert(pTransInst->connType == TAOS_CONN_CLIENT); - // atomic or not - SCliMsg* cliMsg = malloc(sizeof(SCliMsg)); + SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pMsg; cliMsg->st = taosGetTimestampUs(); + cliMsg->type = Normal; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; + + tDebug("send request at thread:%d %p, dst: %s:%d", index, pMsg, ip, port); transSendAsync(thrd->asyncPool, &(cliMsg->q)); } + void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)shandle; int index = CONN_HOST_THREAD_INDEX(pReq->handle); @@ -741,10 +887,11 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq pCtx->pRsp = pRsp; tsem_init(pCtx->pSem, 0, 0); - SCliMsg* cliMsg = malloc(sizeof(SCliMsg)); + SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pReq; cliMsg->st = taosGetTimestampUs(); + cliMsg->type = Normal; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; transSendAsync(thrd->asyncPool, &(cliMsg->q)); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 367cb33fc9b31665cddfb635d46fceaa39cfe929..6c16113d46892e4773b5a43505859017e7bd1e7e 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -155,14 +155,16 @@ bool transReadComplete(SConnBuffer* connBuf) { } return false; } -int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {} +int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) { return 0; } -int transUnpackMsg(STransMsgHead* msgHead) {} +int transUnpackMsg(STransMsgHead* msgHead) { return 0; } int transDestroyBuffer(SConnBuffer* buf) { if (buf->cap > 0) { tfree(buf->buf); } transClearBuffer(buf); + + return 0; } int transSetConnOption(uv_tcp_t* stream) { @@ -184,7 +186,7 @@ SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) SAsyncItem* item = calloc(1, sizeof(SAsyncItem)); item->pThrd = arg; QUEUE_INIT(&item->qmsg); - pthread_mutex_init(&item->mtx, NULL); + taosThreadMutexInit(&item->mtx, NULL); async->data = item; } @@ -195,7 +197,7 @@ void transDestroyAsyncPool(SAsyncPool* pool) { uv_async_t* async = &(pool->asyncs[i]); SAsyncItem* item = async->data; - pthread_mutex_destroy(&item->mtx); + taosThreadMutexDestroy(&item->mtx); free(item); } free(pool->asyncs); @@ -212,9 +214,9 @@ int transSendAsync(SAsyncPool* pool, queue* q) { SAsyncItem* item = async->data; int64_t st = taosGetTimestampUs(); - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_PUSH(&item->qmsg, q); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); int64_t el = taosGetTimestampUs() - st; if (el > 50) { // tInfo("lock and unlock cost: %d", (int)el); @@ -222,4 +224,56 @@ int transSendAsync(SAsyncPool* pool, queue* q) { return uv_async_send(async); } +void transCtxInit(STransCtx* ctx) { + // init transCtx + ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); +} +void transCtxDestroy(STransCtx* ctx) { + if (ctx->args == NULL) { + return; + } + + STransCtxVal* iter = taosHashIterate(ctx->args, NULL); + while (iter) { + iter->free(iter->val); + iter = taosHashIterate(ctx->args, iter); + } + taosHashCleanup(ctx->args); +} + +void transCtxMerge(STransCtx* dst, STransCtx* src) { + if (dst->args == NULL) { + dst->args = src->args; + src->args = NULL; + return; + } + void* key = NULL; + size_t klen = 0; + void* iter = taosHashIterate(src->args, NULL); + while (iter) { + STransCtxVal* sVal = (STransCtxVal*)iter; + key = taosHashGetKey(sVal, &klen); + + STransCtxVal* dVal = taosHashGet(dst->args, key, klen); + if (dVal) { + dVal->free(dVal->val); + } + taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); + iter = taosHashIterate(src->args, iter); + } + taosHashCleanup(src->args); +} +void* transCtxDumpVal(STransCtx* ctx, int32_t key) { + if (ctx->args == NULL) { + return NULL; + } + STransCtxVal* cVal = taosHashGet(ctx->args, (const void*)&key, sizeof(key)); + if (cVal == NULL) { + return NULL; + } + char* ret = calloc(1, cVal->len); + memcpy(ret, (char*)cVal->val, cVal->len); + return (void*)ret; +} + #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index cb3bbaefec847a764e1059f6d9dc1dbc54825d20..3daac3e6f5cbc44250363de1ba2aa98fa9ba1ccb 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -17,6 +17,12 @@ #include "transComm.h" +typedef struct { + int notifyCount; // + int init; // init or not + STransMsg msg; +} SSrvRegArg; + typedef struct SSrvConn { T_REF_DECLARE() uv_tcp_t* pTcp; @@ -33,8 +39,10 @@ typedef struct SSrvConn { void* hostThrd; SArray* srvMsgs; - bool broken; // conn broken; + SSrvRegArg regArg; + bool broken; // conn broken; + ConnStatus status; struct sockaddr_in addr; struct sockaddr_in locaddr; @@ -47,20 +55,20 @@ typedef struct SSrvConn { } SSrvConn; typedef struct SSrvMsg { - SSrvConn* pConn; - STransMsg msg; - queue q; + SSrvConn* pConn; + STransMsg msg; + queue q; + STransMsgType type; } SSrvMsg; typedef struct SWorkThrdObj { - pthread_t thread; - uv_pipe_t* pipe; - uv_os_fd_t fd; - uv_loop_t* loop; - SAsyncPool* asyncPool; - + TdThread thread; + uv_pipe_t* pipe; + uv_os_fd_t fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; queue msg; - pthread_mutex_t msgMtx; + TdThreadMutex msgMtx; queue conn; void* pTransInst; @@ -68,7 +76,7 @@ typedef struct SWorkThrdObj { } SWorkThrdObj; typedef struct SServerObj { - pthread_t thread; + TdThread thread; uv_tcp_t server; uv_loop_t* loop; @@ -85,10 +93,27 @@ typedef struct SServerObj { static const char* notify = "a"; -// refactor later -static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen); - -static int uvAuthMsg(SSrvConn* pConn, char* msg, int msgLen); +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tTrace("server conn %p received release request", conn); \ + \ + STransMsg tmsg = {.handle = (void*)conn, .code = 0}; \ + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); \ + srvMsg->msg = tmsg; \ + srvMsg->type = Release; \ + srvMsg->pConn = conn; \ + taosArrayPush(conn->srvMsgs, &srvMsg); \ + if (taosArrayGetSize(conn->srvMsgs) > 1) { \ + return; \ + } \ + uvStartSendRespInternal(srvMsg); \ + return; \ + } \ + } while (0) static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -106,11 +131,20 @@ static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); static void uvStartSendResp(SSrvMsg* msg); +static void uvNotifyLinkBrokenToApp(SSrvConn* conn); + static void destroySmsg(SSrvMsg* smsg); // check whether already read complete packet static SSrvConn* createConn(void* hThrd); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); +static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd); +static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd); +static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, + uvHandleRegister}; + static void uvDestroyConn(uv_handle_t* handle); // server and worker thread @@ -127,59 +161,6 @@ void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b transAllocBuffer(pBuf, buf); } -static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { - STransMsgHead* pHead = (STransMsgHead*)msg; - - int code = 0; - - if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { - // secured link, or no authentication - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, secured link, no auth is required", pConn->info); - return 0; - } - - if (!rpcIsReq(pHead->msgType)) { - // for response, if code is auth failure, it shall bypass the auth process - code = htonl(pHead->code); - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || - code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); - return 0; - } - } - - code = 0; - if (pHead->spi == pConn->spi) { - // authentication - SRpcDigest* pDigest = (SRpcDigest*)((char*)pHead + len - sizeof(SRpcDigest)); - - int32_t delta; - delta = (int32_t)htonl(pDigest->timeStamp); - delta -= (int32_t)taosGetTimestampSec(); - if (abs(delta) > 900) { - tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); - code = TSDB_CODE_RPC_INVALID_TIME_STAMP; - } else { - if (transAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { - // tDebug("%s, authentication failed, msg discarded", pConn->info); - code = TSDB_CODE_RPC_AUTH_FAILURE; - } else { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); - if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client - // tTrace("%s, message is authenticated", pConn->info); - } - } - } else { - tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); - code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; - } - - return code; -} - // refers specifically to query or insert timeout static void uvHandleActivityTimeout(uv_timer_t* handle) { SSrvConn* conn = handle->data; @@ -187,56 +168,59 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { } static void uvHandleReq(SSrvConn* pConn) { - SRecvInfo info; - SRecvInfo* p = &info; SConnBuffer* pBuf = &pConn->readBuf; - p->msg = pBuf->buf; - p->msgLen = pBuf->len; - p->ip = 0; - p->port = 0; - p->shandle = pConn->pTransInst; // - p->thandle = pConn; - p->chandle = NULL; - - STransMsgHead* pHead = (STransMsgHead*)p->msg; + char* msg = pBuf->buf; + uint32_t msgLen = pBuf->len; + + STransMsgHead* pHead = (STransMsgHead*)msg; if (pHead->secured == 1) { - STransUserMsg* uMsg = (p->msg + p->msgLen - sizeof(STransUserMsg)); + STransUserMsg* uMsg = (STransUserMsg*)((char*)msg + msgLen - sizeof(STransUserMsg)); memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); } pHead->code = htonl(pHead->code); - - int32_t dlen = 0; - if (transDecompressMsg(NULL, 0, NULL)) { - // add compress later - // pHead = rpcDecompresSTransMsg(pHead); - } else { - pHead->msgLen = htonl(pHead->msgLen); - // impl later - // + pHead->msgLen = htonl(pHead->msgLen); + if (pHead->secured == 1) { + pHead->msgLen -= sizeof(STransUserMsg); } + CONN_SHOULD_RELEASE(pConn, pHead); + STransMsg transMsg; transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = pHead->content; transMsg.msgType = pHead->msgType; transMsg.code = pHead->code; transMsg.ahandle = NULL; - transMsg.handle = pConn; + transMsg.handle = NULL; transClearBuffer(&pConn->readBuf); pConn->inType = pHead->msgType; - transRefSrvHandle(pConn); + if (pConn->status == ConnNormal) { + if (pHead->persist == 1) { + pConn->status = ConnAcquire; + transRefSrvHandle(pConn); + } + } + if (pConn->status == ConnNormal && pHead->noResp == 0) { + transRefSrvHandle(pConn); + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port), transMsg.contLen); + } else { + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn, + TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen, pHead->noResp); + // no ref here + } - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), - inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), - ntohs(pConn->locaddr.sin_port), transMsg.contLen); + if (pHead->noResp == 0) { + transMsg.handle = pConn; + } - STrans* pTransInst = (STrans*)p->shandle; - (*((STrans*)p->shandle)->cfp)(pTransInst->parent, &transMsg, NULL); + STrans* pTransInst = pConn->pTransInst; + (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); // uv_timer_start(&pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); - // auth - // validate msg type } void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { @@ -261,13 +245,14 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { tError("server conn %p read error: %s", conn, uv_err_name(nread)); if (nread < 0) { conn->broken = true; + if (conn->status == ConnAcquire) { + if (conn->regArg.init) { + STrans* pTransInst = conn->pTransInst; + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + } transUnrefSrvHandle(conn); - - // if (conn->ref > 1) { - // conn->ref++; // ref > 1 signed that write is in progress - //} - // tError("server conn %p read error: %s", conn, uv_err_name(nread)); - // destroyConn(conn, true); } } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { @@ -289,18 +274,36 @@ void uvOnSendCb(uv_write_t* req, int status) { if (conn->srvMsgs != NULL) { assert(taosArrayGetSize(conn->srvMsgs) >= 1); SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); + tTrace("server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs)); taosArrayRemove(conn->srvMsgs, 0); + if (msg->type == Release && conn->status != ConnNormal) { + conn->status = ConnNormal; + transUnrefSrvHandle(conn); + } destroySmsg(msg); - // send second data, just use for push if (taosArrayGetSize(conn->srvMsgs) > 0) { + tTrace("resent server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs)); msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); - uvStartSendRespInternal(msg); + if (msg->type == Register && conn->status == ConnAcquire) { + conn->regArg.notifyCount = 0; + conn->regArg.init = 1; + conn->regArg.msg = msg->msg; + if (conn->broken) { + STrans* pTransInst = conn->pTransInst; + (pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + taosArrayRemove(conn->srvMsgs, 0); + free(msg); + } else { + uvStartSendRespInternal(msg); + } } } } else { tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); - conn->broken = false; + conn->broken = true; transUnrefSrvHandle(conn); } } @@ -314,7 +317,6 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { - // impl later; tTrace("server conn %p prepare to send resp", smsg->pConn); SSrvConn* pConn = smsg->pConn; @@ -325,20 +327,27 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - pHead->secured = pMsg->code == 0 ? 1 : 0; // - pHead->msgType = smsg->pConn->inType + 1; + // pHead->secured = pMsg->code == 0 ? 1 : 0; // + if (!pConn->secured) { + pConn->secured = pMsg->code == 0 ? 1 : 0; + } + pHead->secured = pConn->secured; + + if (pConn->status == ConnNormal) { + pHead->msgType = pConn->inType + 1; + } else { + pHead->msgType = smsg->type == Release ? 0 : pMsg->msgType; + } + pHead->release = smsg->type == Release ? 1 : 0; pHead->code = htonl(pMsg->code); - // add more info + char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); - if (transCompressMsg(msg, len, NULL)) { - // impl later - } tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), - inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); - pHead->msgLen = htonl(len); + wb->base = msg; wb->len = len; } @@ -356,21 +365,24 @@ static void uvStartSendResp(SSrvMsg* smsg) { SSrvConn* pConn = smsg->pConn; if (pConn->broken == true) { + // persist by transUnrefSrvHandle(pConn); return; } - transUnrefSrvHandle(pConn); + if (pConn->status == ConnNormal) { + transUnrefSrvHandle(pConn); + } - if (taosArrayGetSize(pConn->srvMsgs) > 0) { - tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); - taosArrayPush(pConn->srvMsgs, &smsg); + taosArrayPush(pConn->srvMsgs, &smsg); + if (taosArrayGetSize(pConn->srvMsgs) > 1) { + tDebug("server conn %p send data to client %s:%d, local info: %s:%d", pConn, taosInetNtoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); return; } - taosArrayPush(pConn->srvMsgs, &smsg); uvStartSendRespInternal(smsg); return; } + static void destroySmsg(SSrvMsg* smsg) { if (smsg == NULL) { return; @@ -385,6 +397,9 @@ static void destroyAllConn(SWorkThrdObj* pThrd) { QUEUE_INIT(h); SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue); + while (T_REF_VAL_GET(c) >= 2) { + transUnrefSrvHandle(c); + } transUnrefSrvHandle(c); } } @@ -395,9 +410,9 @@ void uvWorkerAsyncCb(uv_async_t* handle) { queue wq; // batch process to avoid to lock/unlock frequently - pthread_mutex_lock(&item->mtx); + taosThreadMutexLock(&item->mtx); QUEUE_MOVE(&item->qmsg, &wq); - pthread_mutex_unlock(&item->mtx); + taosThreadMutexUnlock(&item->mtx); while (!QUEUE_IS_EMPTY(&wq)) { queue* head = QUEUE_HEAD(&wq); @@ -408,20 +423,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) { tError("unexcept occurred, continue"); continue; } - if (msg->pConn == NULL) { - free(msg); - bool noConn = QUEUE_IS_EMPTY(&pThrd->conn); - if (noConn == true) { - uv_loop_close(pThrd->loop); - uv_stop(pThrd->loop); - } else { - destroyAllConn(pThrd); - // uv_loop_close(pThrd->loop); - pThrd->quit = true; - } - } else { - uvStartSendResp(msg); - } + (*transAsyncHandle[msg->type])(msg, pThrd); } } static void uvAcceptAsyncCb(uv_async_t* async) { @@ -538,6 +540,8 @@ void* acceptThread(void* arg) { setThreadName("trans-accept"); SServerObj* srv = (SServerObj*)arg; uv_run(srv->loop, UV_RUN_DEFAULT); + + return NULL; } static bool addHandleToWorkloop(void* arg) { SWorkThrdObj* pThrd = arg; @@ -552,7 +556,7 @@ static bool addHandleToWorkloop(void* arg) { pThrd->pipe->data = pThrd; QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); + taosThreadMutexInit(&pThrd->msgMtx, NULL); // conn set QUEUE_INIT(&pThrd->conn); @@ -593,6 +597,8 @@ void* workerThread(void* arg) { setThreadName("trans-worker"); SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; uv_run(pThrd->loop, UV_RUN_DEFAULT); + + return NULL; } static SSrvConn* createConn(void* hThrd) { @@ -603,9 +609,11 @@ static SSrvConn* createConn(void* hThrd) { QUEUE_PUSH(&pThrd->conn, &pConn->queue); pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // - tTrace("conn %p created", pConn); + tTrace("server conn %p created", pConn); + memset(&pConn->regArg, 0, sizeof(pConn->regArg)); pConn->broken = false; + pConn->status = ConnNormal; transRefSrvHandle(pConn); return pConn; @@ -623,7 +631,7 @@ static void destroyConn(SSrvConn* conn, bool clear) { } conn->srvMsgs = taosArrayDestroy(conn->srvMsgs); if (clear) { - tTrace("try to destroy conn %p", conn); + tTrace("server conn %p to be destroyed", conn); uv_shutdown_t* req = malloc(sizeof(uv_shutdown_t)); uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } @@ -639,32 +647,13 @@ static void uvDestroyConn(uv_handle_t* handle) { uv_timer_stop(&conn->pTimer); QUEUE_REMOVE(&conn->queue); free(conn->pTcp); - free(conn); + // free(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { uv_loop_close(thrd->loop); uv_stop(thrd->loop); } } -static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen) { - STransMsgHead* pHead = (STransMsgHead*)msg; - - if (pConn->spi && pConn->secured == 0) { - // add auth part - pHead->spi = pConn->spi; - STransDigestMsg* pDigest = (STransDigestMsg*)(msg + msgLen); - pDigest->timeStamp = htonl(taosGetTimestampSec()); - msgLen += sizeof(SRpcDigest); - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); - // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); - } else { - pHead->spi = 0; - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - } - - return msgLen; -} void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SServerObj* srv = calloc(1, sizeof(SServerObj)); @@ -697,7 +686,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, if (false == addHandleToWorkloop(thrd)) { goto End; } - int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); + int err = taosThreadCreate(&(thrd->thread), NULL, workerThread, (void*)(thrd)); if (err == 0) { tDebug("sucess to create worker-thread %d", i); // printf("thread %d create\n", i); @@ -709,7 +698,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, if (false == addHandleToAcceptloop(srv)) { goto End; } - int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); + int err = taosThreadCreate(&srv->thread, NULL, acceptThread, (void*)srv); if (err == 0) { tDebug("success to create accept-thread"); } else { @@ -721,21 +710,70 @@ End: transCloseServer(srv); return NULL; } - +void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { + if (QUEUE_IS_EMPTY(&thrd->conn)) { + uv_loop_close(thrd->loop); + uv_stop(thrd->loop); + } else { + destroyAllConn(thrd); + thrd->quit = true; + } + free(msg); +} +void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) { + // release handle to rpc init + SSrvConn* conn = msg->pConn; + if (conn->status == ConnAcquire) { + taosArrayPush(conn->srvMsgs, &msg); + if (taosArrayGetSize(conn->srvMsgs) > 1) { + return; + } + uvStartSendRespInternal(msg); + return; + } else if (conn->status == ConnRelease || conn->status == ConnNormal) { + tDebug("server conn %p already released, ignore release-msg", conn); + } + destroySmsg(msg); +} +void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd) { + // send msg to client + tDebug("server conn %p start to send resp", msg->pConn); + uvStartSendResp(msg); +} +void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd) { + SSrvConn* conn = msg->pConn; + tDebug("server conn %p register brokenlink callback", conn); + if (conn->status == ConnAcquire) { + if (taosArrayGetSize(conn->srvMsgs) > 0) { + taosArrayPush(conn->srvMsgs, &msg); + return; + } + conn->regArg.notifyCount = 0; + conn->regArg.init = 1; + conn->regArg.msg = msg->msg; + + if (conn->broken) { + STrans* pTransInst = conn->pTransInst; + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); + memset(&conn->regArg, 0, sizeof(conn->regArg)); + } + free(msg); + } +} void destroyWorkThrd(SWorkThrdObj* pThrd) { if (pThrd == NULL) { return; } - pthread_join(pThrd->thread, NULL); + taosThreadJoin(pThrd->thread, NULL); free(pThrd->loop); transDestroyAsyncPool(pThrd->asyncPool); free(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); - tDebug("send quit msg to work thread"); - - transSendAsync(pThrd->asyncPool, &srvMsg->q); + SSrvMsg* msg = calloc(1, sizeof(SSrvMsg)); + msg->type = Quit; + tDebug("server send quit msg to work thread"); + transSendAsync(pThrd->asyncPool, &msg->q); } void transCloseServer(void* arg) { @@ -748,7 +786,7 @@ void transCloseServer(void* arg) { tDebug("send quit msg to accept thread"); uv_async_send(srv->pAcceptAsync); - pthread_join(srv->thread, NULL); + taosThreadJoin(srv->thread, NULL); free(srv->pThreadObj); free(srv->pAcceptAsync); @@ -777,12 +815,28 @@ void transUnrefSrvHandle(void* handle) { return; } int ref = T_REF_DEC((SSrvConn*)handle); - tDebug("handle %p ref count: %d", handle, ref); - + tDebug("server conn %p ref count: %d", handle, ref); if (ref == 0) { destroyConn((SSrvConn*)handle, true); } - // unref srv handle +} + +void transReleaseSrvHandle(void* handle) { + if (handle == NULL) { + return; + } + SSrvConn* pConn = handle; + SWorkThrdObj* pThrd = pConn->hostThrd; + + STransMsg tmsg = {.handle = handle, .code = 0}; + + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + srvMsg->msg = tmsg; + srvMsg->type = Release; + srvMsg->pConn = pConn; + + tTrace("server conn %p start to release", pConn); + transSendAsync(pThrd->asyncPool, &srvMsg->q); } void transSendResponse(const STransMsg* pMsg) { if (pMsg->handle == NULL) { @@ -794,9 +848,24 @@ void transSendResponse(const STransMsg* pMsg) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; + srvMsg->type = Normal; tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); } +void transRegisterMsg(const STransMsg* msg) { + if (msg->handle == NULL) { + return; + } + SSrvConn* pConn = msg->handle; + SWorkThrdObj* pThrd = pConn->hostThrd; + + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + srvMsg->pConn = pConn; + srvMsg->msg = *msg; + srvMsg->type = Register; + tTrace("server conn %p start to register brokenlink callback", pConn); + transSendAsync(pThrd->asyncPool, &srvMsg->q); +} int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) { SSrvConn* pConn = thandle; struct sockaddr_in addr = pConn->addr; diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index c6ff2480efa9c44a5ba9a6526c1934c67baf6cb8..4af316acae25b833c165656aa00cc2dd50ddb750 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -30,7 +30,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t * pOverSem; - pthread_t thread; + TdThread thread; void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -186,8 +186,8 @@ int main(int argc, char *argv[]) { SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < appThreads; ++i) { pInfo->index = i; @@ -196,7 +196,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 1fe13a35b1bd33e17f3b6a452d497caedf0e899c..f3be4e34524ae13f86ca92b9bc60e0b92f032fc6 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -29,7 +29,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t *pOverSem; - pthread_t thread; + TdThread thread; void *pRpc; } SInfo; @@ -80,7 +80,7 @@ int main(int argc, char *argv[]) { char secret[TSDB_KEY_LEN] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.numOfEps = 1; @@ -163,8 +163,8 @@ int main(int argc, char *argv[]) { SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i=0; iindex = i; @@ -173,7 +173,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index b3c2f6ebdc6de3352259e52bf93322bfc2eb01ea..14f32df0b8034a8c54b42583d9f9c380fbb11a70 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -30,7 +30,7 @@ typedef struct { int msgSize; tsem_t rspSem; tsem_t * pOverSem; - pthread_t thread; + TdThread thread; void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -105,7 +105,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - pthread_attr_t thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -186,8 +186,8 @@ int main(int argc, char *argv[]) { SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < appThreads; ++i) { pInfo->index = i; @@ -196,7 +196,7 @@ int main(int argc, char *argv[]) { pInfo->msgSize = msgSize; tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; } diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index fa20327003d82095857fe4cf60c3e3b5840c1d0d..0b1b1834dfb0fae47e666f7806705a6658816b8e 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -15,6 +15,7 @@ #include #include #include +#include "rpcLog.h" #include "tdatablock.h" #include "tglobal.h" #include "tlog.h" @@ -29,7 +30,13 @@ const char *ckey = "ckey"; class Server; int port = 7000; // server process +// server except + typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); + +static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); // client process; static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); @@ -61,22 +68,9 @@ class Client { rpcInit_.cfp = cb; this->transCli = rpcOpen(&rpcInit_); } - void setPersistFP(bool (*pfp)(void *parent, tmsg_t msgType)) { - rpcClose(this->transCli); - rpcInit_.pfp = pfp; - this->transCli = rpcOpen(&rpcInit_); - } - void setConstructFP(void *(*mfp)(void *parent, tmsg_t msgType)) { - rpcClose(this->transCli); - rpcInit_.mfp = mfp; - this->transCli = rpcOpen(&rpcInit_); - } - void setPAndMFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) { + void Stop() { rpcClose(this->transCli); - - rpcInit_.pfp = pfp; - rpcInit_.mfp = mfp; - this->transCli = rpcOpen(&rpcInit_); + this->transCli = NULL; } void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { @@ -88,6 +82,14 @@ class Client { SemWait(); *resp = this->resp; } + void SendAndRecvNoHandle(SRpcMsg *req, SRpcMsg *resp) { + if (req->handle != NULL) { + rpcReleaseHandle(req->handle, TAOS_CONN_CLIENT); + req->handle = NULL; + } + SendAndRecv(req, resp); + } + void SemWait() { tsem_wait(&this->sem); } void SemPost() { tsem_post(&this->sem); } void Reset() {} @@ -105,26 +107,36 @@ class Client { class Server { public: Server() { - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = port; - rpcInit.label = (char *)label; - rpcInit.numOfThreads = 5; - rpcInit.cfp = processReq; - rpcInit.user = (char *)user; - rpcInit.secret = (char *)secret; - rpcInit.ckey = (char *)ckey; - rpcInit.spi = 1; - rpcInit.connType = TAOS_CONN_SERVER; + memset(&rpcInit_, 0, sizeof(rpcInit_)); + rpcInit_.localPort = port; + rpcInit_.label = (char *)label; + rpcInit_.numOfThreads = 5; + rpcInit_.cfp = processReq; + rpcInit_.user = (char *)user; + rpcInit_.secret = (char *)secret; + rpcInit_.ckey = (char *)ckey; + rpcInit_.spi = 1; + rpcInit_.connType = TAOS_CONN_SERVER; } void Start() { - this->transSrv = rpcOpen(&this->rpcInit); + this->transSrv = rpcOpen(&this->rpcInit_); taosMsleep(1000); } + void SetSrvContinueSend(CB cb) { + this->Stop(); + rpcInit_.cfp = cb; + this->Start(); + } void Stop() { if (this->transSrv == NULL) return; rpcClose(this->transSrv); this->transSrv = NULL; } + void SetSrvSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { + this->Stop(); + rpcInit_.cfp = cfp; + this->Start(); + } void Restart() { this->Stop(); this->Start(); @@ -135,7 +147,7 @@ class Server { } private: - SRpcInit rpcInit; + SRpcInit rpcInit_; void * transSrv; }; static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -146,11 +158,52 @@ static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { rpcMsg.code = 0; rpcSendResponse(&rpcMsg); } + +static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + for (int i = 0; i < 10; i++) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + } +} +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + rpcReleaseHandle(pMsg->handle, TAOS_CONN_SERVER); +} +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + void *handle = pMsg->handle; + { + SRpcMsg rpcMsg1 = {0}; + rpcMsg1.pCont = rpcMallocCont(100); + rpcMsg1.contLen = 100; + rpcMsg1.handle = handle; + rpcMsg1.code = 0; + rpcRegisterBrokenLinkArg(&rpcMsg1); + } + taosMsleep(10); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.handle = pMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); +} // client process; static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { Client *client = (Client *)parent; client->SetResp(pMsg); client->SemPost(); + tDebug("received resp"); } static void initEnv() { @@ -170,7 +223,7 @@ static void initEnv() { tsAsyncLog = 0; std::string path = "/tmp/transport"; - taosRemoveDir(path.c_str()); + // taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); tstrncpy(tsLogDir, path.c_str(), PATH_MAX); @@ -178,6 +231,7 @@ static void initEnv() { printf("failed to init log file\n"); } } + class TransObj { public: TransObj() { @@ -188,22 +242,27 @@ class TransObj { srv->Start(); } - void RestartCli(CB cb) { cli->Restart(cb); } - void StopSrv() { srv->Stop(); } - void SetCliPersistFp(bool (*pfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->setPersistFP(pfp); + void RestartCli(CB cb) { + // + cli->Restart(cb); } - void SetCliMFp(void *(*mfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->setConstructFP(mfp); + void StopSrv() { + // + srv->Stop(); } - void SetMAndPFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) { - // do nothing - cli->setPAndMFp(pfp, mfp); + // call when link broken, and notify query or fetch stop + void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { + /////// + srv->SetSrvContinueSend(cfp); } void RestartSrv() { srv->Restart(); } + void StopCli() { + /////// + cli->Stop(); + } void cliSendAndRecv(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecv(req, resp); } + void cliSendAndRecvNoHandle(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecvNoHandle(req, resp); } + ~TransObj() { delete cli; delete srv; @@ -228,7 +287,7 @@ class TransEnv : public ::testing::Test { }; TEST_F(TransEnv, 01sendAndRec) { - for (int i = 0; i < 1; i++) { + for (int i = 0; i < 10; i++) { SRpcMsg req = {0}, resp = {0}; req.msgType = 0; req.pCont = rpcMallocCont(10); @@ -256,29 +315,165 @@ TEST_F(TransEnv, 02StopServer) { tr->cliSendAndRecv(&req, &resp); assert(resp.code != 0); } -TEST_F(TransEnv, clientUserDefined) {} +TEST_F(TransEnv, clientUserDefined) { + tr->RestartSrv(); + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 0; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + assert(resp.code == 0); + } + + ////////////////// +} TEST_F(TransEnv, cliPersistHandle) { - // impl late + SRpcMsg resp = {0}; + void * handle = NULL; + for (int i = 0; i < 10; i++) { + SRpcMsg req = {.handle = resp.handle, .persistHandle = 1}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + // if (i == 5) { + // std::cout << "stop server" << std::endl; + // tr->StopSrv(); + //} + // if (i >= 6) { + // EXPECT_TRUE(resp.code != 0); + //} + handle = resp.handle; + } + rpcReleaseHandle(handle, TAOS_CONN_CLIENT); + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + } + + taosMsleep(1000); + ////////////////// } -TEST_F(TransEnv, srvPersistHandle) { - // impl later + +TEST_F(TransEnv, srvReleaseHandle) { + SRpcMsg resp = {0}; + tr->SetSrvContinueSend(processReleaseHandleCb); + // tr->Restart(processReleaseHandleCb); + void *handle = NULL; + for (int i = 0; i < 1; i++) { + SRpcMsg req = {.handle = resp.handle, .persistHandle = 1}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + // tr->cliSendAndRecvNoHandle(&req, &resp); + EXPECT_TRUE(resp.code == 0); + } + ////////////////// +} +TEST_F(TransEnv, cliReleaseHandleExcept) { + SRpcMsg resp = {0}; + for (int i = 0; i < 3; i++) { + SRpcMsg req = {.handle = resp.handle, .persistHandle = 1}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + if (i == 1) { + std::cout << "stop server" << std::endl; + tr->StopSrv(); + } + if (i > 1) { + EXPECT_TRUE(resp.code != 0); + } + } + ////////////////// +} +TEST_F(TransEnv, srvContinueSend) { + tr->SetSrvContinueSend(processContinueSend); + for (int i = 0; i < 10; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + } + taosMsleep(1000); } -TEST_F(TransEnv, srvPersisHandleExcept) { - // conn breken +TEST_F(TransEnv, srvPersistHandleExcept) { + tr->SetSrvContinueSend(processContinueSend); + // tr->SetCliPersistFp(cliPersistHandle); + SRpcMsg resp = {0}; + for (int i = 0; i < 5; i++) { + SRpcMsg req = {.handle = resp.handle}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + if (i > 2) { + tr->StopCli(); + break; + } + } + taosMsleep(2000); + // conn broken // } -TEST_F(TransEnv, cliPersisHandleExcept) { - // conn breken +TEST_F(TransEnv, cliPersistHandleExcept) { + tr->SetSrvContinueSend(processContinueSend); + SRpcMsg resp = {0}; + for (int i = 0; i < 5; i++) { + SRpcMsg req = {.handle = resp.handle}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + if (i > 2) { + tr->StopSrv(); + break; + } + } + taosMsleep(2000); + // conn broken + // } -TEST_F(TransEnv, multiCliPersisHandleExcept) { - // conn breken -} -TEST_F(TransEnv, multiSrvPersisHandleExcept) { - // conn breken +TEST_F(TransEnv, multiCliPersistHandleExcept) { + // conn broken } TEST_F(TransEnv, queryExcept) { - // query and conn is broken + tr->SetSrvContinueSend(processRegisterFailure); + SRpcMsg resp = {0}; + for (int i = 0; i < 5; i++) { + SRpcMsg req = {.handle = resp.handle, .persistHandle = 1}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + if (i == 2) { + rpcReleaseHandle(resp.handle, TAOS_CONN_CLIENT); + tr->StopCli(); + break; + } + } + taosMsleep(4 * 1000); +} +TEST_F(TransEnv, noResp) { + SRpcMsg resp = {0}; + for (int i = 0; i < 5; i++) { + SRpcMsg req = {.noResp = 1}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendAndRecv(&req, &resp); + } + taosMsleep(2000); + + // no resp } diff --git a/source/libs/transport/test/transportTests.cc b/source/libs/transport/test/transportTests.cc index 53910aa30c1d7bcea1ce0d6dc46f1bb579c3c3c1..1f8c8e8ff20d3ae57d120e02dfd04c8e5a84850d 100644 --- a/source/libs/transport/test/transportTests.cc +++ b/source/libs/transport/test/transportTests.cc @@ -136,4 +136,98 @@ TEST_F(QueueEnv, testIter) { assert(result.size() == vals.size()); } +class TransCtxEnv : public ::testing::Test { + protected: + virtual void SetUp() { + ctx = (STransCtx *)calloc(1, sizeof(STransCtx)); + transCtxInit(ctx); + // TODO + } + virtual void TearDown() { + transCtxDestroy(ctx); + // formate + } + STransCtx *ctx; +}; + +TEST_F(TransCtxEnv, mergeTest) { + int key = 1; + { + STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = malloc(12); + val1.len = 12; + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = malloc(12); + val1.len = 12; + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + free(src); + } + EXPECT_EQ(2, taosHashGetSize(ctx->args)); + { + STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = malloc(12); + val1.len = 12; + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = malloc(12); + val1.len = 12; + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + free(src); + } + std::string val("Hello"); + EXPECT_EQ(4, taosHashGetSize(ctx->args)); + { + key = 1; + STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + transCtxInit(src); + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = calloc(1, 11); + memcpy(val1.val, val.c_str(), val.size()); + val1.len = 11; + + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + { + STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; + val1.val = calloc(1, 11); + memcpy(val1.val, val.c_str(), val.size()); + val1.len = 11; + taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); + key++; + } + transCtxMerge(ctx, src); + free(src); + } + EXPECT_EQ(4, taosHashGetSize(ctx->args)); + + char *skey = (char *)transCtxDumpVal(ctx, 1); + EXPECT_EQ(0, strcmp(skey, val.c_str())); + free(skey); + + skey = (char *)transCtxDumpVal(ctx, 2); + EXPECT_EQ(0, strcmp(skey, val.c_str())); +} #endif diff --git a/source/libs/transport/test/uv.c b/source/libs/transport/test/uv.c index 4c7d30900b220c5b1ea87fb55ffb16a415541986..5a19b7998c915343792cfc4051358eb763b0f863 100644 --- a/source/libs/transport/test/uv.c +++ b/source/libs/transport/test/uv.c @@ -1,17 +1,16 @@ #include -#include +#include #include #include #include #include "task.h" -#include #define NUM_OF_THREAD 1 #define TIMEOUT 10000 typedef struct SThreadObj { - pthread_t thread; + TdThread thread; uv_pipe_t *pipe; uv_loop_t *loop; uv_async_t *workerAsync; // @@ -183,7 +182,7 @@ int main() { server->pThreadObj[i]->fd = fds[0]; server->pThreadObj[i]->pipe = &(server->pipe[i][1]); // init read - int err = pthread_create(&(server->pThreadObj[i]->thread), NULL, + int err = taosThreadCreate(&(server->pThreadObj[i]->thread), NULL, worker_thread, (void *)(server->pThreadObj[i])); if (err == 0) { printf("thread %d create\n", i); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 248f758787d277f0b319d6e89f09a1fde925b80f..70ec7e06557d75358f2b4e33a8aec06b493e06d2 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -19,9 +19,6 @@ #include "tref.h" #include "walInt.h" -#include -#include - int64_t inline walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } int64_t inline walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index b1caeab3b1396ae601326217c956133d776d54f8..2da8f4f8af46ff9156ec61ab5de3d42f2bfc4f4c 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -25,7 +25,7 @@ typedef struct { int8_t inited; uint32_t seq; int32_t refSetId; - pthread_t thread; + TdThread thread; } SWalMgmt; static SWalMgmt tsWal = {0, .seq = 1}; @@ -101,7 +101,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.headVer = WAL_HEAD_VER; pWal->writeHead.magic = WAL_MAGIC; - if (pthread_mutex_init(&pWal->mutex, NULL) < 0) { + if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; @@ -109,7 +109,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->refId = taosAddRef(tsWal.refSetId, pWal); if (pWal->refId < 0) { - pthread_mutex_destroy(&pWal->mutex); + taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; @@ -119,7 +119,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (walCheckAndRepairMeta(pWal) < 0) { taosRemoveRef(tsWal.refSetId, pWal->refId); - pthread_mutex_destroy(&pWal->mutex); + taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; @@ -156,7 +156,7 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { } void walClose(SWal *pWal) { - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); taosCloseFile(&pWal->pWriteLogTFile); pWal->pWriteLogTFile = NULL; taosCloseFile(&pWal->pWriteIdxTFile); @@ -164,7 +164,7 @@ void walClose(SWal *pWal) { walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } @@ -173,7 +173,7 @@ static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); - pthread_mutex_destroy(&pWal->mutex); + taosThreadMutexDestroy(&pWal->mutex); tfree(pWal); } @@ -223,17 +223,17 @@ static void *walThreadFunc(void *param) { } static int32_t walCreateThread() { - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { + if (taosThreadCreate(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { wError("failed to create wal thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread)); return 0; @@ -243,7 +243,7 @@ static void walStopThread() { atomic_store_8(&tsWal.stop, 1); if (taosCheckPthreadValid(tsWal.thread)) { - pthread_join(tsWal.thread, NULL); + taosThreadJoin(tsWal.thread, NULL); } wDebug("wal thread is stopped"); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9e1ffeae7f83deb4692ba221ad3f334fcc6965a2..8d3acef9e74646629284d96a783847088525a1f1 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -169,7 +169,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } if (pRead->pHead->head.version != ver) { - wError("unexpected wal log version: %ld, read request version:%ld", pRead->pHead->head.version, ver); + wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 0f6ac0b214741eab7f365603457479ff778cf876..8392e66e58107914021b14dae9b690d4d24ccf09 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -38,7 +38,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); // find correct file if (ver < walGetLastFileFirstVer(pWal)) { @@ -65,20 +65,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // TODO:change to deserialize function if (pIdxTFile == NULL) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code < 0) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } // read idx file and get log file pos // TODO:change to deserialize function SWalIdxEntry entry; if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } ASSERT(entry.ver == ver); @@ -87,13 +87,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); if (pLogTFile == NULL) { // TODO - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { // TODO - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return -1; } // validate offset @@ -127,7 +127,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; // unlock - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return 0; } @@ -283,7 +283,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in ASSERT(pWal->writeCur >= 0); - pthread_mutex_lock(&pWal->mutex); + taosThreadMutexLock(&pWal->mutex); if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { walSetWrite(pWal); @@ -327,7 +327,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; - pthread_mutex_unlock(&pWal->mutex); + taosThreadMutexUnlock(&pWal->mutex); return 0; } diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 27906c262753f2bf02bd9cd09b4ca9227ba26bba..eea390391126cd86725e39e7c0e756243780ab7f 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -3,10 +3,16 @@ add_library(os STATIC ${OS_SRC}) target_include_directories( os PUBLIC "${CMAKE_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_SOURCE_DIR}/include" - PRIVATE "${CMAKE_SOURCE_DIR}/include/util" - PRIVATE "${CMAKE_SOURCE_DIR}/contrib/pthread-win32" + PUBLIC "${CMAKE_SOURCE_DIR}/include" + PUBLIC "${CMAKE_SOURCE_DIR}/include/util" + PUBLIC "${CMAKE_SOURCE_DIR}/contrib/pthread" + PUBLIC "${CMAKE_SOURCE_DIR}/contrib/gnuregex" ) +# iconv +find_path(IconvApiIncludes iconv.h PATHS) +if(NOT IconvApiIncludes) + add_definitions(-DDISALLOW_NCHAR_WITHOUT_ICONV) +endif () target_link_libraries( os pthread dl rt m ) diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 63fa60021752686ea80a591160c6b7e2f65ae2a8..61b2593bc6c3115f0d5f89c92e593082ad05fcf2 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -101,6 +101,8 @@ void osUpdate() { } } +void osCleanup() {} + bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 6472202bf98df8999e0fb4d188b652b0356c0864..4dece8abefa6c0d25d0dfd0376590a47506d6538 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -14,6 +14,7 @@ */ #define ALLOW_FORBID_FUNC #include "os.h" +#include "osSemaphore.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #include @@ -35,26 +36,35 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ #else #include #include -#include + +#if !defined(_TD_DARWIN_64) + #include +#endif #include #include #define LINUX_FILE_NO_TEXT_OPTION 0 #define O_TEXT LINUX_FILE_NO_TEXT_OPTION #endif +#if defined(WINDOWS) +typedef int32_t FileFd; +typedef int32_t SocketFd; +#else typedef int32_t FileFd; +typedef int32_t SocketFd; +#endif -#define FILE_WITH_LOCK 1 +typedef int32_t FileFd; typedef struct TdFile { -#if FILE_WITH_LOCK - pthread_rwlock_t rwlock; -#endif + TdThreadRwlock rwlock; int refId; FileFd fd; FILE *fp; } * TdFilePtr, TdFile; +#define FILE_WITH_LOCK 1 + void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) const char *tdengineTmpFileNamePrefix = "tdengine-"; @@ -265,7 +275,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { return NULL; } #if FILE_WITH_LOCK - pthread_rwlock_init(&(pFile->rwlock), NULL); + taosThreadRwlockInit(&(pFile->rwlock), NULL); #endif pFile->fd = fd; pFile->fp = fp; @@ -282,7 +292,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_wrlock(&((*ppFile)->rwlock)); + taosThreadRwlockWrlock(&((*ppFile)->rwlock)); #endif if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { return 0; @@ -299,8 +309,8 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { } (*ppFile)->refId = 0; #if FILE_WITH_LOCK - pthread_rwlock_unlock(&((*ppFile)->rwlock)); - pthread_rwlock_destroy(&((*ppFile)->rwlock)); + taosThreadRwlockUnlock(&((*ppFile)->rwlock)); + taosThreadRwlockDestroy(&((*ppFile)->rwlock)); #endif free(*ppFile); *ppFile = NULL; @@ -313,7 +323,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t leftbytes = count; @@ -327,13 +337,13 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { continue; } else { #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return -1; } } else if (readbytes == 0) { #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return (int64_t)(count - leftbytes); } @@ -343,7 +353,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return count; } @@ -353,12 +363,12 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t ret = pread(pFile->fd, buf, count, offset); #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return ret; } @@ -368,7 +378,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_wrlock(&(pFile->rwlock)); + taosThreadRwlockWrlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); @@ -383,7 +393,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { continue; } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return -1; } @@ -392,7 +402,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return count; } @@ -402,12 +412,12 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return 0; } #if FILE_WITH_LOCK - pthread_rwlock_rdlock(&(pFile->rwlock)); + taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); int64_t ret = lseek(pFile->fd, (long)offset, whence); #if FILE_WITH_LOCK - pthread_rwlock_unlock(&(pFile->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return ret; } diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index e9d6ed7c5424aff687175391942171f57c43c5f8..5f12f9cd3d5a2a75cd4080d1f35cdb892ea8e0b8 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -81,7 +81,7 @@ void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { } if (!taosValidateEncodec(inCharSet)) { - printf("Invalid charset:%s, please set the valid charset in config file", inCharSet); + printf("Invalid charset:%s, please set the valid charset in config file\n", inCharSet); exit(-1); } } diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index e5b506e8d835375205a2ffc18a7242d1e65506a1..b41ef898d40fd38c64501378b568f1b7dbc62bf9 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -24,11 +24,11 @@ #include -bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } +bool taosCheckPthreadValid(TdThread thread) { return thread.p != NULL; } -void taosResetPthread(pthread_t* thread) { thread->p = 0; } +void taosResetPthread(TdThread* thread) { thread->p = 0; } -int64_t taosGetPthreadId(pthread_t thread) { +int64_t taosGetPthreadId(TdThread thread) { #ifdef PTW32_VERSION return pthread_getw32threadid_np(thread); #else @@ -38,7 +38,7 @@ int64_t taosGetPthreadId(pthread_t thread) { int64_t taosGetSelfPthreadId() { return GetCurrentThreadId(); } -bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } +bool taosComparePthread(TdThread first, TdThread second) { return first.p == second.p; } int32_t taosGetPId() { return GetCurrentProcessId(); } @@ -84,10 +84,9 @@ int32_t tsem_wait(tsem_t* sem) { #include #include #include -#include -static pthread_t sem_thread; -static pthread_once_t sem_once; +static TdThread sem_thread; +static TdThreadOnce sem_once; static task_t sem_port; static volatile int sem_inited = 0; static semaphore_t sem_exit; @@ -110,7 +109,7 @@ static void *sem_thread_routine(void *arg) { static void once_init(void) { int r = 0; - r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); + r = taosThreadCreate(&sem_thread, NULL, sem_thread_routine, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); return; @@ -123,8 +122,8 @@ static void once_init(void) { struct tsem_s { #ifdef SEM_USE_PTHREAD - pthread_mutex_t lock; - pthread_cond_t cond; + TdThreadMutex lock; + TdThreadCond cond; volatile int64_t val; #elif defined(SEM_USE_POSIX) size_t id; @@ -151,12 +150,12 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { } #ifdef SEM_USE_PTHREAD - int r = pthread_mutex_init(&p->lock, NULL); + int r = taosThreadMutexInit(&p->lock, NULL); do { if (r) break; - r = pthread_cond_init(&p->cond, NULL); + r = taosThreadCondInit(&p->cond, NULL); if (r) { - pthread_mutex_destroy(&p->lock); + taosThreadMutexDestroy(&p->lock); break; } p->val = value; @@ -186,7 +185,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { abort(); } while (p->sem == SEM_FAILED); #elif defined(SEM_USE_SEM) - pthread_once(&sem_once, once_init); + taosThreadOnce(&sem_once, once_init); if (sem_inited != 1) { fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); errno = ENOMEM; @@ -224,18 +223,18 @@ int tsem_wait(tsem_t *sem) { abort(); } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val -= 1; if (p->val < 0) { - if (pthread_cond_wait(&p->cond, &p->lock)) { + if (taosThreadCondWait(&p->cond, &p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -260,18 +259,18 @@ int tsem_post(tsem_t *sem) { abort(); } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->val += 1; if (p->val <= 0) { - if (pthread_cond_signal(&p->cond)) { + if (taosThreadCondSignal(&p->cond)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -299,20 +298,20 @@ int tsem_destroy(tsem_t *sem) { return 0; } #ifdef SEM_USE_PTHREAD - if (pthread_mutex_lock(&p->lock)) { + if (taosThreadMutexLock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } p->valid = 0; - if (pthread_cond_destroy(&p->cond)) { + if (taosThreadCondDestroy(&p->cond)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - if (pthread_mutex_unlock(&p->lock)) { + if (taosThreadMutexUnlock(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - if (pthread_mutex_destroy(&p->lock)) { + if (taosThreadMutexDestroy(&p->lock)) { fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } @@ -338,23 +337,23 @@ int tsem_destroy(tsem_t *sem) { return 0; } -bool taosCheckPthreadValid(pthread_t thread) { +bool taosCheckPthreadValid(TdThread thread) { uint64_t id = 0; - int r = pthread_threadid_np(thread, &id); + int r = TdThreadhreadid_np(thread, &id); return r ? false : true; } int64_t taosGetSelfPthreadId() { uint64_t id; - pthread_threadid_np(0, &id); + TdThreadhreadid_np(0, &id); return (int64_t)id; } -int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } +int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } -void taosResetPthread(pthread_t *thread) { *thread = NULL; } +void taosResetPthread(TdThread *thread) { *thread = NULL; } -bool taosComparePthread(pthread_t first, pthread_t second) { return pthread_equal(first, second) ? true : false; } +bool taosComparePthread(TdThread first, TdThread second) { return taosThreadEqual(first, second) ? true : false; } int32_t taosGetPId() { return (int32_t)getpid(); } @@ -378,7 +377,7 @@ int32_t taosGetAppName(char *name, int32_t *len) { #include #include -bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } +bool taosCheckPthreadValid(TdThread thread) { return thread != 0; } int64_t taosGetSelfPthreadId() { static __thread int id = 0; @@ -387,9 +386,9 @@ int64_t taosGetSelfPthreadId() { return id; } -int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } -void taosResetPthread(pthread_t* thread) { *thread = 0; } -bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } +int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } +void taosResetPthread(TdThread* thread) { *thread = 0; } +bool taosComparePthread(TdThread first, TdThread second) { return first == second; } int32_t taosGetPId() { return getpid(); } int32_t taosGetAppName(char* name, int32_t* len) { diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 698ceded1694fb44565b2cf717cb369b22e6acf8..e5f83fa167cf48f30366dd467da7f59213050838 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -17,7 +17,7 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if defined(WINDOWS) #include #include #include @@ -37,8 +37,14 @@ #include #include #include -#include #include + +#if defined(DARWIN) + #include + #include "osEok.h" +#else + #include +#endif #endif typedef int32_t SocketFd; @@ -46,7 +52,7 @@ typedef SocketFd EpollFd; typedef struct TdSocketServer { #if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif int refId; SocketFd fd; @@ -54,7 +60,7 @@ typedef struct TdSocketServer { typedef struct TdSocket { #if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif int refId; SocketFd fd; @@ -62,7 +68,7 @@ typedef struct TdSocket { typedef struct TdEpoll { #if SOCKET_WITH_LOCK - pthread_rwlock_t rwlock; + TdThreadRwlock rwlock; #endif int refId; EpollFd fd; @@ -210,7 +216,7 @@ int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { #endif } -#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) +#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) #if defined(_TD_GO_DLL_) uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } #endif @@ -769,7 +775,7 @@ void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = taosThreadSigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { // printf("failed to block SIGPIPE"); } @@ -887,7 +893,7 @@ void taosSetMaskSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); + int32_t rc = taosThreadSigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { // printf("failed to setmask SIGPIPE"); } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 1052d108af43599ae2fb6706388a2ce9b4b2e115..d3d1ab5ddaf0c97ce6d2d8e0f5037509721e5a00 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -13,22 +13,69 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE +#include #include "os.h" -#include "tdef.h" -#include -#include +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV +#include "iconv.h" +#endif + +extern int wcwidth(wchar_t c); +extern int wcswidth(const wchar_t *s, size_t n); int64_t taosStr2int64(const char *str) { char *endptr = NULL; return strtoll(str, &endptr, 10); } -#ifdef USE_LIBICONV -#include "iconv.h" +int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { + for (int32_t i = 0; i < bytes; i += sizeof(TdUcs4)) { + int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i); + int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i); + + if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { + return f1 - f2; + } else if (f1 == 0 && f2 == 0) { + return 0; + } + + if (f1 != f2) { + return f1 - f2; + } + } + + return 0; + +#if 0 + int32_t ucs4_max_len = bytes + 4; + char *f1_mbs = calloc(bytes, 1); + char *f2_mbs = calloc(bytes, 1); + if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { + return -1; + } + if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) { + return -1; + } + int32_t ret = strcmp(f1_mbs, f2_mbs); + free(f1_mbs); + free(f2_mbs); + return ret; +#endif +} + -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { +TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { + assert(malloc_usable_size(target_ucs4)>=len_ucs4*sizeof(TdUcs4)); + return memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4)); +} + +int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { +#ifdef DISALLOW_NCHAR_WITHOUT_ICONV + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + return -1; +#else iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); size_t ucs4_input_len = ucs4_max_len; size_t outLen = ucs4_max_len; @@ -39,14 +86,19 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { iconv_close(cd); return (int32_t)(ucs4_max_len - outLen); +#endif } -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { +bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { +#ifdef DISALLOW_NCHAR_WITHOUT_ICONV + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + return -1; +#else memset(ucs4, 0, ucs4_max_len); iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); size_t ucs4_input_len = mbsLength; size_t outLeft = ucs4_max_len; - if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) { + if (iconv(cd, (char**)&mbs, &ucs4_input_len, (char**)&ucs4, &outLeft) == -1) { iconv_close(cd); return false; } @@ -60,9 +112,14 @@ bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len } return true; +#endif } bool taosValidateEncodec(const char *encodec) { +#ifdef DISALLOW_NCHAR_WITHOUT_ICONV + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + return true; +#else iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); if (cd == (iconv_t)(-1)) { return false; @@ -70,214 +127,11 @@ bool taosValidateEncodec(const char *encodec) { iconv_close(cd); return true; -} - -#else - -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { - mbstate_t state = {0}; - int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state); - if (len < 0) { - return -1; - } - - memset(&state, 0, sizeof(state)); - len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state); - if (len < 0) { - return -1; - } - - return len; -} - -bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { - memset(ucs4, 0, ucs4_max_len); - mbstate_t state = {0}; - int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); - *len = retlen; - - return retlen >= 0; -} - -bool taosValidateEncodec(const char *encodec) { - return true; -} - -#endif - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -/* - * windows implementation - */ - -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include -#include - -#if STDC_HEADERS -#include -#else -char *malloc(), *realloc(); #endif - -/* Always add at least this many bytes when extending the buffer. */ -#define MIN_CHUNK 64 - -/* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR -+ OFFSET (and null-terminate it). *LINEPTR is a pointer returned from -malloc (or NULL), pointing to *N characters of space. It is realloc'd -as necessary. Return the number of characters read (not including the -null terminator), or -1 on error or EOF. On a -1 return, the caller -should check feof(), if not then errno has been set to indicate -the error. */ - -int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t offset) { - int32_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */ - char * read_pos; /* Where we're reading into *LINEPTR. */ - int32_t ret; - - if (!lineptr || !n || !stream) { - errno = EINVAL; - return -1; - } - - if (!*lineptr) { - *n = MIN_CHUNK; - *lineptr = malloc(*n); - if (!*lineptr) { - errno = ENOMEM; - return -1; - } - } - - nchars_avail = (int32_t)(*n - offset); - read_pos = *lineptr + offset; - - for (;;) { - int32_t save_errno; - register int32_t c = getc(stream); - - save_errno = errno; - - /* We always want at least one char left in the buffer, since we - always (unless we get an error while reading the first char) - NUL-terminate the line buffer. */ - - assert((*lineptr + *n) == (read_pos + nchars_avail)); - if (nchars_avail < 2) { - if (*n > MIN_CHUNK) - *n *= 2; - else - *n += MIN_CHUNK; - - nchars_avail = (int32_t)(*n + *lineptr - read_pos); - char* lineptr1 = realloc(*lineptr, *n); - if (!lineptr1) { - errno = ENOMEM; - return -1; - } - *lineptr = lineptr1; - - read_pos = *n - nchars_avail + *lineptr; - assert((*lineptr + *n) == (read_pos + nchars_avail)); - } - - if (ferror(stream)) { - /* Might like to return partial line, but there is no - place for us to store errno. And we don't want to just - lose errno. */ - errno = save_errno; - return -1; - } - - if (c == EOF) { - /* Return partial line, if any. */ - if (read_pos == *lineptr) - return -1; - else - break; - } - - *read_pos++ = c; - nchars_avail--; - - if (c == terminator) /* Return the line. */ - break; - } - - /* Done - NUL terminate and return the number of chars read. */ - *read_pos = '\0'; - - ret = (int32_t)(read_pos - (*lineptr + offset)); - return ret; -} - -int32_t tgetline(char **lineptr, size_t *n, FILE *stream) { return getstr(lineptr, n, stream, '\n', 0); } - - -/* - * Get next token from string *stringp, where tokens are possibly-empty - * strings separated by characters from delim. - * - * Writes NULs into the string at *stringp to end tokens. - * delim need not remain constant from call to call. - * On return, *stringp points past the last NUL written (if there might - * be further tokens), or is NULL (if there are definitely no moretokens). - * - * If *stringp is NULL, strsep returns NULL. - */ -char *strsep(char **stringp, const char *delim) { - char * s; - const char *spanp; - int32_t c, sc; - char *tok; - if ((s = *stringp) == NULL) - return (NULL); - for (tok = s;;) { - c = *s++; - spanp = delim; - do { - if ((sc = *spanp++) == c) { - if (c == 0) - s = NULL; - else - s[-1] = 0; - *stringp = s; - return (tok); - } - } while (sc != 0); - } - /* NOTREACHED */ -} - -char *getpass(const char *prefix) { - static char passwd[TSDB_PASSWORD_LEN] = {0}; - memset(passwd, 0, TSDB_PASSWORD_LEN); - //printf("%s", prefix); - - int32_t index = 0; - char ch; - while (index < TSDB_PASSWORD_LEN) { - ch = getch(); - if (ch == '\n' || ch == '\r') { - break; - } else { - passwd[index++] = ch; - } - } - - return passwd; } -int32_t twcslen(const wchar_t *wcs) { - int32_t *wstr = (int32_t *)wcs; +int32_t taosUcs4len(TdUcs4 *ucs4) { + TdUcs4 *wstr = (TdUcs4 *)ucs4; if (NULL == wstr) { return 0; } @@ -292,73 +146,15 @@ int32_t twcslen(const wchar_t *wcs) { return n; } -int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) { - for (int32_t i = 0; i < bytes; i += TSDB_NCHAR_SIZE) { - int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i); - int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i); - - if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { - return f1 - f2; - } else if (f1 == 0 && f2 == 0) { - return 0; - } - - if (f1 != f2) { - return f1 - f2; - } - } - - return 0; - -#if 0 - int32_t ucs4_max_len = bytes + 4; - char *f1_mbs = calloc(bytes, 1); - char *f2_mbs = calloc(bytes, 1); - if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { - return -1; - } - if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) { - return -1; - } - int32_t ret = strcmp(f1_mbs, f2_mbs); - free(f1_mbs); - free(f2_mbs); - return ret; -#endif -} -/* Copy memory to memory until the specified number of bytes -has been copied, return pointer to following byte. -Overlap is NOT handled correctly. */ -void *mempcpy(void *dest, const void *src, size_t len) { - return (char*)memcpy(dest, src, len) + len; -} +int32_t taosWcharWidth(TdWchar wchar) { return wcwidth(wchar); } -/* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */ -char *stpcpy (char *dest, const char *src) { - size_t len = strlen (src); - return (char*)memcpy(dest, src, len + 1) + len; -} - -/* Copy no more than N characters of SRC to DEST, returning the address of - the terminating '\0' in DEST, if any, or else DEST + N. */ -char *stpncpy (char *dest, const char *src, size_t n) { - size_t size = strnlen (src, n); - memcpy (dest, src, size); - dest += size; - if (size == n) - return dest; - return memset (dest, '\0', n - size); -} +int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { return wcswidth(pWchar, size); } -#else +int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size) { return mbtowc(pWchar, pStr, size); } -/* - * linux and darwin implementation - */ +int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { return mbstowcs(pWchars, pStrs, size); } -int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize) { - return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / ncharSize); -} +int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); } -#endif +int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); } diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c new file mode 100644 index 0000000000000000000000000000000000000000..716080274ba74315d6aef9a7bf40ee70738648aa --- /dev/null +++ b/source/os/src/osThread.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC +#include "os.h" + +// int32_t taosThreadSetnameNp(TdThread thread, const char *name) { +// return pthread_setname_np(thread,name); +// } + +int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared) { + return pthread_spin_init(lock, pshared); +} + +int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) { + return pthread_mutex_init(mutex, attr); +} + +int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { + return pthread_spin_destroy(lock); +} + +int32_t taosThreadMutexDestroy(TdThreadMutex * mutex) { + return pthread_mutex_destroy(mutex); +} + +int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { + return pthread_spin_lock(lock); +} + +int32_t taosThreadMutexLock(TdThreadMutex *mutex) { + return pthread_mutex_lock(mutex); +} + +int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { + return pthread_spin_unlock(lock); +} + +int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { + return pthread_mutex_unlock(mutex); +} + +int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_rdlock(rwlock); +} + +int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_wrlock(rwlock); +} + +int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) { + return pthread_rwlock_unlock(rwlock); +} + +void taosThreadTestCancel(void) { + return pthread_testcancel(); +} + +int32_t taosThreadAttrInit(TdThreadAttr *attr) { + return pthread_attr_init(attr); +} + +int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg) { + return pthread_create(tid, attr, start, arg); +} + +int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void)) { + return pthread_once(onceControl, initRoutine); +} + +int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState) { + return pthread_attr_setdetachstate(attr, detachState); +} + +int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { + return pthread_attr_destroy(attr); +} + +int32_t taosThreadJoin(TdThread thread, void **pValue) { + return pthread_join(thread, pValue); +} + +int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr) { + return pthread_rwlock_init(rwlock, attr); +} + +int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) { + return pthread_rwlock_destroy(rwlock); +} + +int32_t taosThreadCondSignal(TdThreadCond *cond) { + return pthread_cond_signal(cond); +} + +int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) { + return pthread_cond_init(cond, attr); +} + +int32_t taosThreadCondBroadcast(TdThreadCond *cond) { + return pthread_cond_broadcast(cond); +} + +int32_t taosThreadCondDestroy(TdThreadCond *cond) { + return pthread_cond_destroy(cond); +} + +int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { + return pthread_cond_wait(cond, mutex); +} + +TdThread taosThreadSelf(void) { + return pthread_self(); +} + +// int32_t taosThreadGetW32ThreadIdNp(TdThread thread) { +// return pthread_getw32threadid_np(thread); +// } + +int32_t taosThreadEqual(TdThread t1, TdThread t2) { + return pthread_equal(t1, t2); +} + +int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset) { + return pthread_sigmask(how, set, oset); +} + +int32_t taosThreadCancel(TdThread thread) { + return pthread_cancel(thread); +} + +int32_t taosThreadKill(TdThread thread, int sig) { + return pthread_kill(thread, sig); +} \ No newline at end of file diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index c95ca72bd57fd9e8165cda69edcdb003992816c0..e6e3ec7962192a3aca4da49f9bfe9394d7bdc73b 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -44,7 +44,7 @@ static MMRESULT timerId; static void (*timer_callback)(int); static int timer_ms = 0; -static pthread_t timer_thread; +static TdThread timer_thread; static int timer_kq = -1; static volatile int timer_stop = 0; @@ -83,7 +83,7 @@ static void taosDeleteTimer(void *tharg) { timer_delete(*pTimer); } -static pthread_t timerThread; +static TdThread timerThread; static timer_t timerId; static volatile bool stopTimer = false; static void * taosProcessAlarmSignal(void *tharg) { @@ -112,7 +112,7 @@ static void * taosProcessAlarmSignal(void *tharg) { // printf("Failed to create timer"); } - pthread_cleanup_push(taosDeleteTimer, &timerId); + taosThreadCleanupPush(taosDeleteTimer, &timerId); struct itimerspec ts; ts.it_value.tv_sec = 0; @@ -136,7 +136,7 @@ static void * taosProcessAlarmSignal(void *tharg) { callback(0); } - pthread_cleanup_pop(1); + taosThreadCleanupPop(1); return NULL; } @@ -165,7 +165,7 @@ int taosInitTimer(void (*callback)(int), int ms) { abort(); } - r = pthread_create(&timer_thread, NULL, timer_routine, NULL); + r = taosThreadCreate(&timer_thread, NULL, timer_routine, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment @@ -174,10 +174,10 @@ int taosInitTimer(void (*callback)(int), int ms) { return 0; #else stopTimer = false; - pthread_attr_t tattr; - pthread_attr_init(&tattr); - int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback); - pthread_attr_destroy(&tattr); + TdThreadAttr tattr; + taosThreadAttrInit(&tattr); + int code = taosThreadCreate(&timerThread, &tattr, taosProcessAlarmSignal, callback); + taosThreadAttrDestroy(&tattr); if (code != 0) { // printf("failed to create timer thread"); return -1; @@ -195,7 +195,7 @@ void taosUninitTimer() { #elif defined(_TD_DARWIN_64) int r = 0; timer_stop = 1; - r = pthread_join(timer_thread, NULL); + r = taosThreadJoin(timer_thread, NULL); if (r) { fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__); // since no caller of this func checks the return value for the moment @@ -207,7 +207,7 @@ void taosUninitTimer() { stopTimer = true; // printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); - pthread_join(timerThread, NULL); + taosThreadJoin(timerThread, NULL); #endif } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 864a60b706c2283628e36aefe6873a2aeeea5264..bdafa63d643b0a44be0bd1bfdbc7ed53540a584b 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" @@ -48,16 +49,11 @@ void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { if (inTimezone == NULL || inTimezone[0] == 0) return; -#ifdef WINDOWS +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) char winStr[TD_LOCALE_LEN * 2]; sprintf(winStr, "TZ=%s", inTimezone); putenv(winStr); -#else - setenv("TZ", inTimezone, 1); -#endif tzset(); - - /* * get CURRENT time zone. * system current time zone is affected by daylight saving time(DST) * @@ -75,15 +71,34 @@ void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *ou int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); tz += daylight; - /* * format: * (CST, +0800) * (BST, +0100) */ + sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + *outDaylight = daylight; + +#elif defined(_TD_DARWIN_64) + + setenv("TZ", inTimezone, 1); + tzset(); + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + tz += daylight; sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = daylight; + +#else + setenv("TZ", inTimezone, 1); + tzset(); + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + tz += daylight; + sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + *outDaylight = daylight; + +#endif + } void taosGetSystemTimezone(char *outTimezone) { diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 08fa0a240903736007605d349f11b953c4aacc64..7a47639e75f2e18e6ed47a19066c0a69ea336503 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -5,10 +5,6 @@ target_include_directories( util PUBLIC "${CMAKE_SOURCE_DIR}/include/util" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -IF(${TD_WINDOWS}) - PRIVATE "${CMAKE_SOURCE_DIR}/contrib/pthread-win32" - PRIVATE "${CMAKE_SOURCE_DIR}/contrib/gnuregex" -ENDIF () ) target_link_libraries( util diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 51ea2b1f13c6e8fab1a9ecca59dd6d44e51d9c6d..9a1b2135345adacdc4d3a1d475aab18a97abfdbc 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -22,9 +22,9 @@ #define CACHE_MAX_CAPACITY 1024*1024*16 #define CACHE_DEFAULT_CAPACITY 1024*4 -static pthread_t cacheRefreshWorker = {0}; -static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; +static TdThread cacheRefreshWorker = {0}; +static TdThreadOnce cacheThreadInit = PTHREAD_ONCE_INIT; +static TdThreadMutex guard = PTHREAD_MUTEX_INITIALIZER; static SArray *pCacheArrayList = NULL; static bool stopRefreshWorker = false; static bool refreshWorkerNormalStopped = false; @@ -90,13 +90,13 @@ struct SCacheObj { STrashElem *pTrash; uint8_t deleting; // set the deleting flag to stop refreshing ASAP. - pthread_t refreshWorker; + TdThread refreshWorker; bool extendLifespan; // auto extend life span when one item is accessed. int64_t checkTick; // tick used to record the check times of the refresh threads #if defined(LINUX) - pthread_rwlock_t lock; + TdThreadRwlock lock; #else - pthread_mutex_t lock; + TdThreadMutex lock; #endif }; @@ -109,33 +109,33 @@ typedef struct SCacheObjTravSup { static FORCE_INLINE void __trashcan_wr_lock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_wrlock(&pCacheObj->lock); + taosThreadRwlockWrlock(&pCacheObj->lock); #else - pthread_mutex_lock(&pCacheObj->lock); + taosThreadMutexLock(&pCacheObj->lock); #endif } static FORCE_INLINE void __trashcan_unlock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_unlock(&pCacheObj->lock); + taosThreadRwlockUnlock(&pCacheObj->lock); #else - pthread_mutex_unlock(&pCacheObj->lock); + taosThreadMutexUnlock(&pCacheObj->lock); #endif } static FORCE_INLINE int32_t __trashcan_lock_init(SCacheObj *pCacheObj) { #if defined(LINUX) - return pthread_rwlock_init(&pCacheObj->lock, NULL); + return taosThreadRwlockInit(&pCacheObj->lock, NULL); #else - return pthread_mutex_init(&pCacheObj->lock, NULL); + return taosThreadMutexInit(&pCacheObj->lock, NULL); #endif } static FORCE_INLINE void __trashcan_lock_destroy(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_destroy(&pCacheObj->lock); + taosThreadRwlockDestroy(&pCacheObj->lock); #else - pthread_mutex_destroy(&pCacheObj->lock); + taosThreadMutexDestroy(&pCacheObj->lock); #endif } @@ -154,20 +154,20 @@ static void *taosCacheTimedRefresh(void *handle); static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&cacheRefreshWorker, &thattr, taosCacheTimedRefresh, NULL); - pthread_attr_destroy(&thattr); + taosThreadCreate(&cacheRefreshWorker, &thattr, taosCacheTimedRefresh, NULL); + taosThreadAttrDestroy(&thattr); } -pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) { - pthread_once(&cacheThreadInit, doInitRefreshThread); +TdThread doRegisterCacheObj(SCacheObj *pCacheObj) { + taosThreadOnce(&cacheThreadInit, doInitRefreshThread); - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); taosArrayPush(pCacheArrayList, &pCacheObj); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); return cacheRefreshWorker; } @@ -836,19 +836,19 @@ void *taosCacheTimedRefresh(void *handle) { goto _end; } - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); size_t size = taosArrayGetSize(pCacheArrayList); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); count += 1; for (int32_t i = 0; i < size; ++i) { - pthread_mutex_lock(&guard); + taosThreadMutexLock(&guard); SCacheObj *pCacheObj = taosArrayGetP(pCacheArrayList, i); if (pCacheObj == NULL) { uError("object is destroyed. ignore and try next"); - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); continue; } @@ -860,11 +860,11 @@ void *taosCacheTimedRefresh(void *handle) { uDebug("%s is destroying, remove it from refresh list, remain cache obj:%" PRIzu, pCacheObj->name, size); pCacheObj->deleting = 0; // reset the deleting flag to enable pCacheObj to continue releasing resources. - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); continue; } - pthread_mutex_unlock(&guard); + taosThreadMutexUnlock(&guard); if ((count % pCacheObj->checkTick) != 0) { continue; @@ -892,7 +892,7 @@ _end: taosArrayDestroy(pCacheArrayList); pCacheArrayList = NULL; - pthread_mutex_destroy(&guard); + taosThreadMutexDestroy(&guard); refreshWorkerNormalStopped = true; uDebug("cache refresh thread quits"); diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 062d144933136891acc98b4ba07b43d09c8a9f69..ff7d2cf733c6af1043957d83d0dc24a8073db571 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -208,7 +208,7 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { if (len1 != len2) { return len1 > len2 ? 1 : -1; } else { - int32_t ret = memcmp((wchar_t *)pLeft, (wchar_t *)pRight, len1); + int32_t ret = memcmp((TdUcs4 *)pLeft, (TdUcs4 *)pRight, len1); if (ret == 0) { return 0; } else { @@ -295,10 +295,10 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { - wchar_t c, c1; - wchar_t matchOne = L'_'; // "_" - wchar_t matchAll = L'%'; // "%" +int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) { + TdUcs4 c, c1; + TdUcs4 matchOne = L'_'; // "_" + TdUcs4 matchAll = L'%'; // "%" int32_t i = 0; int32_t j = 0; @@ -315,7 +315,7 @@ int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t siz return TSDB_PATTERN_MATCH; } - wchar_t accept[3] = {towupper(c), towlower(c), 0}; + TdUcs4 accept[3] = {towupper(c), towlower(c), 0}; while (1) { size_t n = wcscspn(str, accept); @@ -424,10 +424,10 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); + char *pattern = calloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); + int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); free(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; @@ -647,7 +647,7 @@ int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) { if (t1->len != t2->len) { return t1->len > t2->len ? 1 : -1; } - int32_t ret = memcmp((wchar_t *)t1, (wchar_t *)t2, t2->len); + int32_t ret = memcmp((TdUcs4 *)t1, (TdUcs4 *)t2, t2->len); if (ret == 0) { return ret; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index d4bcc27f6094867270b84caa0ab9cbc163dbfe31..404c9c8f7184b937fa28add3794868397a257085 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -570,6 +570,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: + case CFG_DTYPE_NONE: if (dump) { printf("%s %s %s", src, name, pItem->str); printf("\n"); @@ -655,4 +656,4 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { uInfo("load from apoll url %s", url); return 0; -} \ No newline at end of file +} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 08180a1c49bd68ae420a6e37c558c14f4f6177be..b7ab007cf55e36d0af45ffee8652264e7a105b18 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -84,6 +84,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") +TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM, "Out of Share memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") @@ -273,33 +274,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with aggregatio TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OFFLINE, "Dnode is offline") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_READ_FILE_ERROR, "Read dnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR, "Write dnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED, "Mnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_NOT_DEPLOYED, "Mnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_INVALID_OPTION, "Mnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_READ_FILE_ERROR, "Read mnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR, "Write mnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED, "Qnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_NOT_DEPLOYED, "Qnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_INVALID_OPTION, "Qnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_READ_FILE_ERROR, "Read qnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR, "Write qnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED, "Snode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_NOT_DEPLOYED, "Snode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_INVALID_OPTION, "Snode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_READ_FILE_ERROR, "Read snode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR, "Write snode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED, "Bnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_NOT_DEPLOYED, "Bnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_INVALID_OPTION, "Bnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_READ_FILE_ERROR, "Read bnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR, "Write bnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_ALREADY_DEPLOYED, "Node already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_NOT_DEPLOYED, "Node not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_PARSE_FILE_ERROR, "Invalid json format") +TAOS_DEFINE_ERROR(TSDB_CODE_NODE_INVALID_OPTION, "Invalid node option") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED, "Vnode already deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_NOT_DEPLOYED, "Vnode not deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_INVALID_OPTION, "Vnode option invalid") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR, "Read vnodes.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR, "Write vnodes.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnodes") // vnode @@ -352,6 +333,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag valu TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") @@ -458,14 +440,14 @@ static int32_t taosCompareTaosError(const void* a, const void* b) { return 0; } -static pthread_once_t tsErrorInit = PTHREAD_ONCE_INIT; +static TdThreadOnce tsErrorInit = PTHREAD_ONCE_INIT; static void tsSortError(void) { qsort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError); } const char* tstrerror(int32_t err) { - pthread_once(&tsErrorInit, tsSortError); + taosThreadOnce(&tsErrorInit, tsSortError); // this is a system errno if ((err & 0x00ff0000) == 0x00ff0000) { diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index 3ae537eae8a9b7c5b16bc91af5c994c140179ee4..705cb0d2d3c7cd97a60c850843e30604a7995a08 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -31,7 +31,7 @@ void *taosInitIdPool(int32_t maxId) { pIdPool->numOfFree = maxId; pIdPool->freeSlot = 0; - pthread_mutex_init(&pIdPool->mutex, NULL); + taosThreadMutexInit(&pIdPool->mutex, NULL); uDebug("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); @@ -42,7 +42,7 @@ int32_t taosAllocateId(id_pool_t *pIdPool) { if (pIdPool == NULL) return -1; int32_t slot = -1; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); if (pIdPool->numOfFree > 0) { for (int32_t i = 0; i < pIdPool->maxId; ++i) { @@ -56,14 +56,14 @@ int32_t taosAllocateId(id_pool_t *pIdPool) { } } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return slot + 1; } void taosFreeId(id_pool_t *pIdPool, int32_t id) { if (pIdPool == NULL) return; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t slot = (id - 1) % pIdPool->maxId; if (pIdPool->freeList[slot]) { @@ -71,7 +71,7 @@ void taosFreeId(id_pool_t *pIdPool, int32_t id) { pIdPool->numOfFree++; } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); } void taosIdPoolCleanUp(id_pool_t *pIdPool) { @@ -81,7 +81,7 @@ void taosIdPoolCleanUp(id_pool_t *pIdPool) { if (pIdPool->freeList) free(pIdPool->freeList); - pthread_mutex_destroy(&pIdPool->mutex); + taosThreadMutexDestroy(&pIdPool->mutex); memset(pIdPool, 0, sizeof(id_pool_t)); @@ -89,16 +89,16 @@ void taosIdPoolCleanUp(id_pool_t *pIdPool) { } int32_t taosIdPoolNumOfUsed(id_pool_t *pIdPool) { - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t ret = pIdPool->maxId - pIdPool->numOfFree; - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } bool taosIdPoolMarkStatus(id_pool_t *pIdPool, int32_t id) { bool ret = false; - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t slot = (id - 1) % pIdPool->maxId; if (!pIdPool->freeList[slot]) { @@ -109,7 +109,7 @@ bool taosIdPoolMarkStatus(id_pool_t *pIdPool, int32_t id) { ret = false; } - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } @@ -123,7 +123,7 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { return -1; } - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); @@ -133,15 +133,15 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { pIdPool->freeList = idList; free(oldIdList); - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return 0; } int32_t taosIdPoolMaxSize(id_pool_t *pIdPool) { - pthread_mutex_lock(&pIdPool->mutex); + taosThreadMutexLock(&pIdPool->mutex); int32_t ret = pIdPool->maxId; - pthread_mutex_unlock(&pIdPool->mutex); + taosThreadMutexUnlock(&pIdPool->mutex); return ret; } \ No newline at end of file diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 634cfcb026eb7b34c350886f43ededa2dc11f885..a85da8cbbf56f6a69f582ecc1e34469786f7cc6d 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -26,6 +26,14 @@ SJson* tjsonCreateObject() { return pJson; } +SJson* tjsonCreateArray() { + SJson* pJson = cJSON_CreateArray(); + if (pJson == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return pJson; +} + void tjsonDelete(SJson* pJson) { if (pJson != NULL) { cJSON_Delete((cJSON*)pJson); @@ -194,9 +202,16 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV return (errno == ERANGE||errno == EINVAL) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; } +int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal) { + uint64_t val = 0; + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + *pVal = val; + return code; +} + int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) { uint64_t val = 0; - int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); *pVal = val; return code; } @@ -231,6 +246,22 @@ int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, voi return func(pJsonObj, pObj); } +int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize) { + if (objSize <= 0) { + return TSDB_CODE_SUCCESS; + } + + SJson* pJsonObj = tjsonGetObjectItem(pJson, pName); + if (NULL == pJsonObj) { + return TSDB_CODE_FAILED; + } + *pObj = calloc(1, objSize); + if (NULL == *pObj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return func(pJsonObj, *pObj); +} + int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize) { const cJSON* jArray = tjsonGetObjectItem(pJson, pName); int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray)); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 6dedc3f7406d3653c1513cb345b68cd17c966e1e..85db7883cf6b18ea05eb07be1faf90fc8672cc76 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -45,8 +45,8 @@ typedef struct { int32_t minBuffSize; TdFilePtr pFile; int32_t stop; - pthread_t asyncThread; - pthread_mutex_t buffMutex; + TdThread asyncThread; + TdThreadMutex buffMutex; tsem_t buffNotEmpty; } SLogBuff; @@ -59,7 +59,7 @@ typedef struct { pid_t pid; char logName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; - pthread_mutex_t logMutex; + TdThreadMutex logMutex; } SLogObj; static int8_t tsLogInited = 0; @@ -107,12 +107,12 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { - pthread_attr_t threadAttr; - pthread_attr_init(&threadAttr); - if (pthread_create(&(tsLogObj.logHandle->asyncThread), &threadAttr, taosAsyncOutputLog, tsLogObj.logHandle) != 0) { + TdThreadAttr threadAttr; + taosThreadAttrInit(&threadAttr); + if (taosThreadCreate(&(tsLogObj.logHandle->asyncThread), &threadAttr, taosAsyncOutputLog, tsLogObj.logHandle) != 0) { return -1; } - pthread_attr_destroy(&threadAttr); + taosThreadAttrDestroy(&threadAttr); return 0; } @@ -139,8 +139,9 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { - pthread_join(tsLogObj.logHandle->asyncThread, NULL); + taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); } + tsLogInited = 0; // In case that other threads still use log resources causing invalid write in valgrind // we comment two lines below. // taosLogBuffDestroy(tsLogObj.logHandle); @@ -223,22 +224,22 @@ static void *taosThreadToOpenNewFile(void *param) { } static int32_t taosOpenNewLogFile() { - pthread_mutex_lock(&tsLogObj.logMutex); + taosThreadMutexLock(&tsLogObj.logMutex); if (tsLogObj.lines > tsLogObj.maxLines && tsLogObj.openInProgress == 0) { tsLogObj.openInProgress = 1; uInfo("open new log file ......"); - pthread_t thread; - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + TdThread thread; + TdThreadAttr attr; + taosThreadAttrInit(&attr); + taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thread, &attr, taosThreadToOpenNewFile, NULL); - pthread_attr_destroy(&attr); + taosThreadCreate(&thread, &attr, taosThreadToOpenNewFile, NULL); + taosThreadAttrDestroy(&attr); } - pthread_mutex_unlock(&tsLogObj.logMutex); + taosThreadMutexUnlock(&tsLogObj.logMutex); return 0; } @@ -343,7 +344,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { char fileName[LOG_FILE_NAME_LEN + 50] = "\0"; sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag); - pthread_mutex_init(&tsLogObj.logMutex, NULL); + taosThreadMutexInit(&tsLogObj.logMutex, NULL); taosUmaskFile(0); tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); @@ -517,7 +518,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { tLogBuff->minBuffSize = bufSize / 10; tLogBuff->stop = 0; - if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; + if (taosThreadMutexInit(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; // tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); return tLogBuff; @@ -552,7 +553,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms if (tLogBuff == NULL || tLogBuff->stop) return -1; - pthread_mutex_lock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexLock(&LOG_BUF_MUTEX(tLogBuff)); start = LOG_BUF_START(tLogBuff); end = LOG_BUF_END(tLogBuff); @@ -566,7 +567,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms if (remainSize <= msgLen || ((lostLine > 0) && (remainSize <= (msgLen + tmpBufLen)))) { lostLine++; tsAsyncLogLostLines++; - pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexUnlock(&LOG_BUF_MUTEX(tLogBuff)); return -1; } @@ -587,7 +588,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms } */ - pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); + taosThreadMutexUnlock(&LOG_BUF_MUTEX(tLogBuff)); return 0; } diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index 1fc9bfc7ab458dfda7d1e74123bbea2366a70e38..d62e90397765af5c646a8fdfd3c07a04bfdd38ae 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -25,7 +25,7 @@ typedef struct { int32_t blockSize; /* block size in bytes */ int32_t *freeList; /* the index list */ char *pool; /* the actual mem block */ - pthread_mutex_t mutex; + TdThreadMutex mutex; } pool_t; mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { @@ -58,7 +58,7 @@ mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { return NULL; } - pthread_mutex_init(&(pool_p->mutex), NULL); + taosThreadMutexInit(&(pool_p->mutex), NULL); memset(pool_p->pool, 0, (size_t)(blockSize * numOfBlock)); for (i = 0; i < pool_p->numOfBlock; ++i) pool_p->freeList[i] = i; @@ -73,7 +73,7 @@ char *taosMemPoolMalloc(mpool_h handle) { char *pos = NULL; pool_t *pool_p = (pool_t *)handle; - pthread_mutex_lock(&(pool_p->mutex)); + taosThreadMutexLock(&(pool_p->mutex)); if (pool_p->numOfFree > 0) { pos = pool_p->pool + pool_p->blockSize * (pool_p->freeList[pool_p->first]); @@ -82,7 +82,7 @@ char *taosMemPoolMalloc(mpool_h handle) { pool_p->numOfFree--; } - pthread_mutex_unlock(&(pool_p->mutex)); + taosThreadMutexUnlock(&(pool_p->mutex)); if (pos == NULL) uDebug("mempool: out of memory"); return pos; @@ -108,18 +108,18 @@ void taosMemPoolFree(mpool_h handle, char *pMem) { memset(pMem, 0, (size_t)pool_p->blockSize); - pthread_mutex_lock(&pool_p->mutex); + taosThreadMutexLock(&pool_p->mutex); pool_p->freeList[(pool_p->first + pool_p->numOfFree) % pool_p->numOfBlock] = index; pool_p->numOfFree++; - pthread_mutex_unlock(&pool_p->mutex); + taosThreadMutexUnlock(&pool_p->mutex); } void taosMemPoolCleanUp(mpool_h handle) { pool_t *pool_p = (pool_t *)handle; - pthread_mutex_destroy(&pool_p->mutex); + taosThreadMutexDestroy(&pool_p->mutex); if (pool_p->pool) free(pool_p->pool); if (pool_p->freeList) free(pool_p->freeList); memset(pool_p, 0, sizeof(*pool_p)); diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c new file mode 100644 index 0000000000000000000000000000000000000000..f19a17fdb6dc26450801ac8ae1abe6391e355e45 --- /dev/null +++ b/source/util/src/tprocess.c @@ -0,0 +1,460 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "tprocess.h" +#include "taoserror.h" +#include "tlog.h" +#include "tqueue.h" + +// todo +#include +#include + +#define SHM_DEFAULT_SIZE (20 * 1024 * 1024) +#define CEIL8(n) (ceil((float)(n) / 8) * 8) +typedef void *(*ProcThreadFp)(void *param); + +typedef struct SProcQueue { + int32_t head; + int32_t tail; + int32_t total; + int32_t avail; + int32_t items; + char *pBuffer; + ProcMallocFp mallocHeadFp; + ProcFreeFp freeHeadFp; + ProcMallocFp mallocBodyFp; + ProcFreeFp freeBodyFp; + ProcConsumeFp consumeFp; + void *pParent; + tsem_t sem; + TdThreadMutex *mutex; + int32_t mutexShmid; + int32_t bufferShmid; + const char *name; +} SProcQueue; + +typedef struct SProcObj { + TdThread childThread; + SProcQueue *pChildQueue; + TdThread parentThread; + SProcQueue *pParentQueue; + const char *name; + int32_t pid; + bool isChild; + bool stopFlag; + bool testFlag; +} SProcObj; + +static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { + TdThreadMutex *pMutex = NULL; + TdThreadMutexAttr mattr = {0}; + int32_t shmid = -1; + int32_t code = -1; + + if (pthread_mutexattr_init(&mattr) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while init attr since %s", terrstr()); + goto _OVER; + } + + if (pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while set shared since %s", terrstr()); + goto _OVER; + } + + shmid = shmget(IPC_PRIVATE, sizeof(TdThreadMutex), 0600); + if (shmid <= 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while shmget since %s", terrstr()); + goto _OVER; + } + + pMutex = (TdThreadMutex *)shmat(shmid, NULL, 0); + if (pMutex == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex while shmat since %s", terrstr()); + goto _OVER; + } + + if (taosThreadMutexInit(pMutex, &mattr) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init mutex since %s", terrstr()); + goto _OVER; + } + + code = 0; + +_OVER: + if (code != 0) { + taosThreadMutexDestroy(pMutex); + shmctl(shmid, IPC_RMID, NULL); + } else { + *ppMutex = pMutex; + *pShmid = shmid; + } + + pthread_mutexattr_destroy(&mattr); + return code; +} + +static void taosProcDestroyMutex(TdThreadMutex *pMutex, int32_t *pShmid) { + if (pMutex != NULL) { + taosThreadMutexDestroy(pMutex); + } + if (*pShmid > 0) { + shmctl(*pShmid, IPC_RMID, NULL); + } +} + +static int32_t taosProcInitBuffer(void **ppBuffer, int32_t size) { + int32_t shmid = shmget(IPC_PRIVATE, size, IPC_CREAT | 0600); + if (shmid <= 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init buffer while shmget since %s", terrstr()); + return -1; + } + + void *shmptr = shmat(shmid, NULL, 0); + if (shmptr == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init buffer while shmat since %s", terrstr()); + shmctl(shmid, IPC_RMID, NULL); + return -1; + } + + *ppBuffer = shmptr; + return shmid; +} + +static void taosProcDestroyBuffer(void *pBuffer, int32_t *pShmid) { + if (*pShmid > 0) { + shmctl(*pShmid, IPC_RMID, NULL); + } +} + +static SProcQueue *taosProcQueueInit(int32_t size) { + if (size <= 0) size = SHM_DEFAULT_SIZE; + + int32_t bufSize = CEIL8(size); + int32_t headSize = CEIL8(sizeof(SProcQueue)); + + SProcQueue *pQueue = NULL; + int32_t shmId = taosProcInitBuffer((void **)&pQueue, bufSize + headSize); + if (shmId <= 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pQueue->bufferShmid = shmId; + + if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { + free(pQueue); + return NULL; + } + + if (tsem_init(&pQueue->sem, 1, 0) != 0) { + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + free(pQueue); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + tsem_destroy(&pQueue->sem); + free(pQueue); + return NULL; + } + + pQueue->head = 0; + pQueue->tail = 0; + pQueue->total = bufSize; + pQueue->avail = bufSize; + pQueue->items = 0; + pQueue->pBuffer = (char *)pQueue + headSize; + return pQueue; +} + +static void taosProcQueueCleanup(SProcQueue *pQueue) { + if (pQueue != NULL) { + uDebug("proc:%s, queue:%p clean up", pQueue->name, pQueue); + taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); + tsem_destroy(&pQueue->sem); + free(pQueue); + } +} + +static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHeadLen, char *pBody, int32_t rawBodyLen) { + const int32_t headLen = CEIL8(rawHeadLen); + const int32_t bodyLen = CEIL8(rawBodyLen); + const int32_t fullLen = headLen + bodyLen + 8; + + taosThreadMutexLock(pQueue->mutex); + if (fullLen > pQueue->avail) { + taosThreadMutexUnlock(pQueue->mutex); + terrno = TSDB_CODE_OUT_OF_SHM_MEM; + return -1; + } + + if (pQueue->tail < pQueue->total) { + *(int32_t *)(pQueue->pBuffer + pQueue->head) = headLen; + *(int32_t *)(pQueue->pBuffer + pQueue->head + 4) = bodyLen; + } else { + *(int32_t *)(pQueue->pBuffer) = headLen; + *(int32_t *)(pQueue->pBuffer + 4) = bodyLen; + } + + if (pQueue->tail < pQueue->head) { + memcpy(pQueue->pBuffer + pQueue->tail + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->tail + 8 + headLen, pBody, rawBodyLen); + pQueue->tail = pQueue->tail + 8 + headLen + bodyLen; + } else { + int32_t remain = pQueue->total - pQueue->tail; + if (remain == 0) { + memcpy(pQueue->pBuffer + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + 8 + headLen, pBody, rawBodyLen); + pQueue->tail = 8 + headLen + bodyLen; + } else if (remain == 8) { + memcpy(pQueue->pBuffer, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + headLen, pBody, rawBodyLen); + pQueue->tail = headLen + bodyLen; + } else if (remain < 8 + headLen) { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, remain - 8); + memcpy(pQueue->pBuffer, pHead + remain - 8, rawHeadLen - (remain - 8)); + memcpy(pQueue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); + pQueue->tail = headLen - (remain - 8) + bodyLen; + } else if (remain < 8 + bodyLen) { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->head + 8 + headLen, pBody, remain - 8 - headLen); + memcpy(pQueue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); + pQueue->tail = bodyLen - (remain - 8 - headLen); + } else { + memcpy(pQueue->pBuffer + pQueue->head + 8, pHead, rawHeadLen); + memcpy(pQueue->pBuffer + pQueue->head + headLen + 8, pBody, rawBodyLen); + pQueue->tail = pQueue->head + headLen + bodyLen + 8; + } + } + + pQueue->avail -= fullLen; + pQueue->items++; + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + + uTrace("proc:%s, push msg:%p:%d cont:%p:%d to queue:%p", pQueue->name, pHead, rawHeadLen, pBody, rawBodyLen, pQueue); + return 0; +} + +static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHeadLen, void **ppBody, + int32_t *pBodyLen) { + tsem_wait(&pQueue->sem); + + taosThreadMutexLock(pQueue->mutex); + if (pQueue->total - pQueue->avail <= 0) { + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + terrno = TSDB_CODE_OUT_OF_SHM_MEM; + return -1; + } + + int32_t headLen = 0; + int32_t bodyLen = 0; + if (pQueue->head < pQueue->total) { + headLen = *(int32_t *)(pQueue->pBuffer + pQueue->head); + bodyLen = *(int32_t *)(pQueue->pBuffer + pQueue->head + 4); + } else { + headLen = *(int32_t *)(pQueue->pBuffer); + bodyLen = *(int32_t *)(pQueue->pBuffer + 4); + } + + void *pHead = (*pQueue->mallocHeadFp)(headLen); + void *pBody = (*pQueue->mallocBodyFp)(bodyLen); + if (pHead == NULL || pBody == NULL) { + taosThreadMutexUnlock(pQueue->mutex); + tsem_post(&pQueue->sem); + (*pQueue->freeHeadFp)(pHead); + (*pQueue->freeBodyFp)(pBody); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (pQueue->head < pQueue->tail) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, bodyLen); + pQueue->head = pQueue->head + 8 + headLen + bodyLen; + } else { + int32_t remain = pQueue->total - pQueue->head; + if (remain == 0) { + memcpy(pHead, pQueue->pBuffer + 8, headLen); + memcpy(pBody, pQueue->pBuffer + 8 + headLen, bodyLen); + pQueue->head = 8 + headLen + bodyLen; + } else if (remain == 8) { + memcpy(pHead, pQueue->pBuffer, headLen); + memcpy(pBody, pQueue->pBuffer + headLen, bodyLen); + pQueue->head = headLen + bodyLen; + } else if (remain < 8 + headLen) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, remain - 8); + memcpy(pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); + memcpy(pBody, pQueue->pBuffer + headLen - (remain - 8), bodyLen); + pQueue->head = headLen - (remain - 8) + bodyLen; + } else if (remain < 8 + bodyLen) { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, remain - 8 - headLen); + memcpy(pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); + pQueue->head = bodyLen - (remain - 8 - headLen); + } else { + memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); + memcpy(pBody, pQueue->pBuffer + pQueue->head + headLen + 8, bodyLen); + pQueue->head = pQueue->head + headLen + bodyLen + 8; + } + } + + pQueue->avail = pQueue->avail + headLen + bodyLen + 8; + pQueue->items--; + taosThreadMutexUnlock(pQueue->mutex); + + *ppHead = pHead; + *ppBody = pBody; + *pHeadLen = headLen; + *pBodyLen = bodyLen; + + uTrace("proc:%s, get msg:%p:%d cont:%p:%d from queue:%p", pQueue->name, pHead, headLen, pBody, bodyLen, pQueue); + return 0; +} + +SProcObj *taosProcInit(const SProcCfg *pCfg) { + SProcObj *pProc = calloc(1, sizeof(SProcObj)); + if (pProc == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pProc->name = pCfg->name; + pProc->testFlag = pCfg->testFlag; + + pProc->pChildQueue = taosProcQueueInit(pCfg->childQueueSize); + pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); + if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { + taosProcQueueCleanup(pProc->pChildQueue); + free(pProc); + return NULL; + } + + pProc->pChildQueue->name = pCfg->name; + pProc->pChildQueue->pParent = pCfg->pParent; + pProc->pChildQueue->mallocHeadFp = pCfg->childMallocHeadFp; + pProc->pChildQueue->freeHeadFp = pCfg->childFreeHeadFp; + pProc->pChildQueue->mallocBodyFp = pCfg->childMallocBodyFp; + pProc->pChildQueue->freeBodyFp = pCfg->childFreeBodyFp; + pProc->pChildQueue->consumeFp = pCfg->childConsumeFp; + pProc->pParentQueue->name = pCfg->name; + pProc->pParentQueue->pParent = pCfg->pParent; + pProc->pParentQueue->mallocHeadFp = pCfg->parentdMallocHeadFp; + pProc->pParentQueue->freeHeadFp = pCfg->parentFreeHeadFp; + pProc->pParentQueue->mallocBodyFp = pCfg->parentMallocBodyFp; + pProc->pParentQueue->freeBodyFp = pCfg->parentFreeBodyFp; + pProc->pParentQueue->consumeFp = pCfg->parentConsumeFp; + + uDebug("proc:%s, initialized, child queue:%p parent queue:%p", pProc->name, pProc->pChildQueue, pProc->pParentQueue); + + if (!pProc->testFlag) { + pProc->pid = fork(); + if (pProc->pid == 0) { + pProc->isChild = 1; + uInfo("this is child process, pid:%d", pProc->pid); + } else { + pProc->isChild = 0; + uInfo("this is parent process, pid:%d", pProc->pid); + } + } + + return pProc; +} + +static void taosProcThreadLoop(SProcQueue *pQueue) { + ProcConsumeFp consumeFp = pQueue->consumeFp; + void *pParent = pQueue->pParent; + void *pHead, *pBody; + int32_t headLen, bodyLen; + + uDebug("proc:%s, start to get message from queue:%p", pQueue->name, pQueue); + + while (1) { + int32_t code = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen); + if (code < 0) { + uDebug("proc:%s, get no message from queue:%p and exiting", pQueue->name, pQueue); + break; + } else if (code < 0) { + uTrace("proc:%s, get no message from queue:%p since %s", pQueue->name, pQueue, terrstr()); + taosMsleep(1); + continue; + } else { + (*consumeFp)(pParent, pHead, headLen, pBody, bodyLen); + } + } +} + +int32_t taosProcRun(SProcObj *pProc) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pProc->isChild || pProc->testFlag) { + if (taosThreadCreate(&pProc->childThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pChildQueue) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create thread since %s", terrstr()); + return -1; + } + uDebug("proc:%s, child start to consume queue:%p", pProc->name, pProc->pChildQueue); + } + + if (!pProc->isChild || pProc->testFlag) { + if (taosThreadCreate(&pProc->parentThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pParentQueue) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create thread since %s", terrstr()); + return -1; + } + uDebug("proc:%s, parent start to consume queue:%p", pProc->name, pProc->pParentQueue); + } + + return 0; +} + +void taosProcStop(SProcObj *pProc) { + pProc->stopFlag = true; + // todo join +} + +bool taosProcIsChild(SProcObj *pProc) { return pProc->isChild; } + +void taosProcCleanup(SProcObj *pProc) { + if (pProc != NULL) { + uDebug("proc:%s, clean up", pProc->name); + taosProcStop(pProc); + taosProcQueueCleanup(pProc->pChildQueue); + taosProcQueueCleanup(pProc->pParentQueue); + free(pProc); + } +} + +int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) { + return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen); +} + +int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) { + return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen); +} diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 000352203348ec8ea0ff3d587c8de89557d994dd..99675630f49ceabfa7ffb0e84f067d5f2ab073b1 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -37,13 +37,13 @@ typedef struct STaosQueue { void *ahandle; // for queue set FItem itemFp; FItems itemsFp; - pthread_mutex_t mutex; + TdThreadMutex mutex; } STaosQueue; typedef struct STaosQset { STaosQueue *head; STaosQueue *current; - pthread_mutex_t mutex; + TdThreadMutex mutex; int32_t numOfQueues; int32_t numOfItems; tsem_t sem; @@ -63,7 +63,7 @@ STaosQueue *taosOpenQueue() { return NULL; } - if (pthread_mutex_init(&queue->mutex, NULL) != 0) { + if (taosThreadMutexInit(&queue->mutex, NULL) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -84,11 +84,11 @@ void taosCloseQueue(STaosQueue *queue) { STaosQnode *pTemp; STaosQset *qset; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); STaosQnode *pNode = queue->head; queue->head = NULL; qset = queue->qset; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (queue->qset) { taosRemoveFromQset(qset, queue); @@ -100,7 +100,7 @@ void taosCloseQueue(STaosQueue *queue) { free(pTemp); } - pthread_mutex_destroy(&queue->mutex); + taosThreadMutexDestroy(&queue->mutex); free(queue); uDebug("queue:%p is closed", queue); @@ -110,19 +110,19 @@ bool taosQueueEmpty(STaosQueue *queue) { if (queue == NULL) return true; bool empty = false; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head == NULL && queue->tail == NULL) { empty = true; } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return empty; } int32_t taosQueueSize(STaosQueue *queue) { - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); int32_t numOfItems = queue->numOfItems; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return numOfItems; } @@ -151,7 +151,7 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { STaosQnode *pNode = (STaosQnode *)(((char *)pItem) - sizeof(STaosQnode)); pNode->next = NULL; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->tail) { queue->tail->next = pNode; @@ -165,7 +165,7 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); uTrace("item:%p is put into queue:%p, items:%d", pItem, queue, queue->numOfItems); - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (queue->qset) tsem_post(&queue->qset->sem); @@ -176,7 +176,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { STaosQnode *pNode = NULL; int32_t code = 0; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { pNode = queue->head; @@ -189,7 +189,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return code; } @@ -202,7 +202,7 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { int32_t code = 0; bool empty; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); empty = queue->head == NULL; if (!empty) { @@ -219,7 +219,7 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, qall->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); // if source queue is empty, we set destination qall to empty too. if (empty) { @@ -255,7 +255,7 @@ STaosQset *taosOpenQset() { return NULL; } - pthread_mutex_init(&qset->mutex, NULL); + taosThreadMutexInit(&qset->mutex, NULL); tsem_init(&qset->sem, 0, 0); uDebug("qset:%p is opened", qset); @@ -266,7 +266,7 @@ void taosCloseQset(STaosQset *qset) { if (qset == NULL) return; // remove all the queues from qset - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); while (qset->head) { STaosQueue *queue = qset->head; qset->head = qset->head->next; @@ -274,9 +274,9 @@ void taosCloseQset(STaosQset *qset) { queue->qset = NULL; queue->next = NULL; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); - pthread_mutex_destroy(&qset->mutex); + taosThreadMutexDestroy(&qset->mutex); tsem_destroy(&qset->sem); free(qset); uDebug("qset:%p is closed", qset); @@ -293,19 +293,19 @@ void taosQsetThreadResume(STaosQset *qset) { int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { if (queue->qset) return -1; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); queue->next = qset->head; queue->ahandle = ahandle; qset->head = queue; qset->numOfQueues++; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); atomic_add_fetch_32(&qset->numOfItems, queue->numOfItems); queue->qset = qset; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); uTrace("queue:%p is added into qset:%p", queue, qset); return 0; @@ -314,7 +314,7 @@ int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { STaosQueue *tqueue = NULL; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); if (qset->head) { if (qset->head == queue) { @@ -339,15 +339,15 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { if (qset->current == queue) qset->current = tqueue->next; qset->numOfQueues--; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); atomic_sub_fetch_32(&qset->numOfItems, queue->numOfItems); queue->qset = NULL; queue->next = NULL; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); } } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); uDebug("queue:%p is removed from qset:%p", queue, qset); } @@ -360,7 +360,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI tsem_wait(&qset->sem); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; @@ -369,7 +369,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI if (queue == NULL) break; if (queue->head == NULL) continue; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { pNode = queue->head; @@ -385,11 +385,11 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (pNode) break; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return code; } @@ -399,7 +399,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand int32_t code = 0; tsem_wait(&qset->sem); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; @@ -408,7 +408,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand if (queue == NULL) break; if (queue->head == NULL) continue; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { qall->current = queue->head; @@ -428,12 +428,12 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand } } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (code != 0) break; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return code; } @@ -443,7 +443,7 @@ int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **aha tsem_wait(&qset->sem); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; @@ -456,7 +456,7 @@ int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **aha continue; } - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); if (queue->head) { pNode = queue->head; @@ -475,11 +475,11 @@ int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **aha uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); if (pNode) break; } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return code; } @@ -488,21 +488,21 @@ void taosResetQsetThread(STaosQset *qset, void *pItem) { if (pItem == NULL) return; STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); pNode->queue->threadId = -1; for (int32_t i = 0; i < pNode->queue->numOfItems; ++i) { tsem_post(&qset->sem); } - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); } int32_t taosGetQueueItemsNumber(STaosQueue *queue) { if (!queue) return 0; int32_t num; - pthread_mutex_lock(&queue->mutex); + taosThreadMutexLock(&queue->mutex); num = queue->numOfItems; - pthread_mutex_unlock(&queue->mutex); + taosThreadMutexUnlock(&queue->mutex); return num; } @@ -510,8 +510,8 @@ int32_t taosGetQsetItemsNumber(STaosQset *qset) { if (!qset) return 0; int32_t num = 0; - pthread_mutex_lock(&qset->mutex); + taosThreadMutexLock(&qset->mutex); num = qset->numOfItems; - pthread_mutex_unlock(&qset->mutex); + taosThreadMutexUnlock(&qset->mutex); return num; } diff --git a/source/util/src/tref.c b/source/util/src/tref.c index a9f6c21bf80ad48f34c895e25c46074fe013d7cd..86633fe07a9769867b1992e00611e072aae1cff0 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -45,8 +45,8 @@ typedef struct { } SRefSet; static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; -static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t tsRefMutex; +static TdThreadOnce tsRefModuleInit = PTHREAD_ONCE_INIT; +static TdThreadMutex tsRefMutex; static int32_t tsRefSetNum = 0; static int32_t tsNextId = 0; @@ -63,7 +63,7 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { int64_t *lockedBy; int32_t i, rsetId; - pthread_once(&tsRefModuleInit, taosInitRefModule); + taosThreadOnce(&tsRefModuleInit, taosInitRefModule); nodeList = calloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { @@ -78,7 +78,7 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { return -1; } - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; @@ -107,7 +107,7 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); return rsetId; } @@ -124,7 +124,7 @@ int32_t taosCloseRef(int32_t rsetId) { pSet = tsRefSetList + rsetId; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); if (pSet->state == TSDB_REF_STATE_ACTIVE) { pSet->state = TSDB_REF_STATE_DELETED; @@ -134,7 +134,7 @@ int32_t taosCloseRef(int32_t rsetId) { uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); if (deleted) taosDecRsetCount(pSet); @@ -354,7 +354,7 @@ int32_t taosListRef() { SRefNode *pNode; int32_t num = 0; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); for (int32_t i = 0; i < TSDB_REF_OBJECTS; ++i) { pSet = tsRefSetList + i; @@ -374,7 +374,7 @@ int32_t taosListRef() { } } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); return num; } @@ -470,7 +470,7 @@ static void taosUnlockList(int64_t *lockedBy) { } } -static void taosInitRefModule(void) { pthread_mutex_init(&tsRefMutex, NULL); } +static void taosInitRefModule(void) { taosThreadMutexInit(&tsRefMutex, NULL); } static void taosIncRsetCount(SRefSet *pSet) { atomic_add_fetch_32(&pSet->count, 1); @@ -483,7 +483,7 @@ static void taosDecRsetCount(SRefSet *pSet) { if (count > 0) return; - pthread_mutex_lock(&tsRefMutex); + taosThreadMutexLock(&tsRefMutex); if (pSet->state != TSDB_REF_STATE_EMPTY) { pSet->state = TSDB_REF_STATE_EMPTY; @@ -497,5 +497,5 @@ static void taosDecRsetCount(SRefSet *pSet) { uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count); } - pthread_mutex_unlock(&tsRefMutex); + taosThreadMutexUnlock(&tsRefMutex); } diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 740e742bad5e9b9f9e50194bcc02aadc1146cba3..316c1f7ec138677e5125186760c6c747c804d8e2 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -26,12 +26,12 @@ typedef struct { char label[TSDB_LABEL_LEN]; tsem_t emptySem; tsem_t fullSem; - pthread_mutex_t queueMutex; + TdThreadMutex queueMutex; int32_t fullSlot; int32_t emptySlot; int32_t queueSize; int32_t numOfThreads; - pthread_t *qthread; + TdThread *qthread; SSchedMsg *queue; bool stop; void *pTmrCtrl; @@ -55,7 +55,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab return NULL; } - pSched->qthread = calloc(sizeof(pthread_t), numOfThreads); + pSched->qthread = calloc(sizeof(TdThread), numOfThreads); if (pSched->qthread == NULL) { uError("%s: no enough memory for qthread", label); taosCleanUpScheduler(pSched); @@ -68,7 +68,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab pSched->fullSlot = 0; pSched->emptySlot = 0; - if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) { + if (taosThreadMutexInit(&pSched->queueMutex, NULL) < 0) { uError("init %s:queueMutex failed(%s)", label, strerror(errno)); taosCleanUpScheduler(pSched); return NULL; @@ -88,11 +88,11 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab pSched->stop = false; for (int32_t i = 0; i < numOfThreads; ++i) { - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int32_t code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); - pthread_attr_destroy(&attr); + TdThreadAttr attr; + taosThreadAttrInit(&attr); + taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_JOINABLE); + int32_t code = taosThreadCreate(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); + taosThreadAttrDestroy(&attr); if (code != 0) { uError("%s: failed to create rpc thread(%s)", label, strerror(errno)); taosCleanUpScheduler(pSched); @@ -135,7 +135,7 @@ void *taosProcessSchedQueue(void *scheduler) { break; } - if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -144,7 +144,7 @@ void *taosProcessSchedQueue(void *scheduler) { memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg)); pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize; - if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -177,7 +177,7 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { exit(ret); } - if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -185,7 +185,7 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { pSched->queue[pSched->emptySlot] = *pMsg; pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize; - if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); exit(ret); } @@ -208,13 +208,13 @@ void taosCleanUpScheduler(void *param) { } for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { - pthread_join(pSched->qthread[i], NULL); + taosThreadJoin(pSched->qthread[i], NULL); } } tsem_destroy(&pSched->emptySem); tsem_destroy(&pSched->fullSem); - pthread_mutex_destroy(&pSched->queueMutex); + taosThreadMutexDestroy(&pSched->queueMutex); if (pSched->pTimer) { taosTmrStopA(&pSched->pTimer); diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index b5e54d12d380aea8edb55f3e69a675913b6f46b2..f7e56b6f3161aadff69d629dcbbfb305be94d49c 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -70,13 +70,13 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ } if (SL_IS_THREAD_SAFE(pSkipList)) { - pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t)); + pSkipList->lock = (TdThreadRwlock *)calloc(1, sizeof(TdThreadRwlock)); if (pSkipList->lock == NULL) { tSkipListDestroy(pSkipList); return NULL; } - if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { + if (taosThreadRwlockInit(pSkipList->lock, NULL) != 0) { tSkipListDestroy(pSkipList); return NULL; } @@ -109,7 +109,7 @@ void tSkipListDestroy(SSkipList *pSkipList) { tSkipListUnlock(pSkipList); if (pSkipList->lock != NULL) { - pthread_rwlock_destroy(pSkipList->lock); + taosThreadRwlockDestroy(pSkipList->lock); tfree(pSkipList->lock); } @@ -435,21 +435,21 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t static FORCE_INLINE int32_t tSkipListWLock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_wrlock(pSkipList->lock); + return taosThreadRwlockWrlock(pSkipList->lock); } return 0; } static FORCE_INLINE int32_t tSkipListRLock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_rdlock(pSkipList->lock); + return taosThreadRwlockRdlock(pSkipList->lock); } return 0; } static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList) { if (pSkipList->lock) { - return pthread_rwlock_unlock(pSkipList->lock); + return taosThreadRwlockUnlock(pSkipList->lock); } return 0; } diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index f9e28d7b6256bb61a51f8c76bc85da338c4f9d60..da76135b6899f4526d533b6d6faaca9e2f15261f 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -16,13 +16,13 @@ #define _DEFAULT_SOURCE #include "tthread.h" -pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { - pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int32_t ret = pthread_create(pthread, &thattr, __start_routine, param); - pthread_attr_destroy(&thattr); +TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param) { + TdThread* pthread = (TdThread*)malloc(sizeof(TdThread)); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + int32_t ret = taosThreadCreate(pthread, &thattr, __start_routine, param); + taosThreadAttrDestroy(&thattr); if (ret != 0) { free(pthread); @@ -31,20 +31,20 @@ pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { return pthread; } -bool taosDestoryThread(pthread_t* pthread) { +bool taosDestoryThread(TdThread* pthread) { if (pthread == NULL) return false; if (taosThreadRunning(pthread)) { - pthread_cancel(*pthread); - pthread_join(*pthread, NULL); + taosThreadCancel(*pthread); + taosThreadJoin(*pthread, NULL); } free(pthread); return true; } -bool taosThreadRunning(pthread_t* pthread) { +bool taosThreadRunning(TdThread* pthread) { if (pthread == NULL) return false; - int32_t ret = pthread_kill(*pthread, 0); + int32_t ret = taosThreadKill(*pthread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 7bdcf3cc6449fa11dd9e903134b2a593fb48ee0e..cc31d9c94a2f083d01c96b8672ace3540188b16a 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -102,7 +102,7 @@ typedef struct timer_map_t { } timer_map_t; typedef struct time_wheel_t { - pthread_mutex_t mutex; + TdThreadMutex mutex; int64_t nextScanAt; uint32_t resolution; uint16_t size; @@ -112,8 +112,8 @@ typedef struct time_wheel_t { int32_t tsMaxTmrCtrl = 512; -static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t tmrCtrlMutex; +static TdThreadOnce tmrModuleInit = PTHREAD_ONCE_INIT; +static TdThreadMutex tmrCtrlMutex; static tmr_ctrl_t* tmrCtrls; static tmr_ctrl_t* unusedTmrCtrl = NULL; static void* tmrQhandle; @@ -230,7 +230,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { timer->prev = NULL; timer->expireAt = taosGetMonotonicMs() + delay; - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); uint32_t idx = 0; if (timer->expireAt > wheel->nextScanAt) { @@ -248,7 +248,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { p->prev = timer; } - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); } static bool removeFromWheel(tmr_obj_t* timer) { @@ -259,7 +259,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { time_wheel_t* wheel = wheels + wheelIdx; bool removed = false; - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); // other thread may modify timer->wheel, check again. if (timer->wheel < tListLen(wheels)) { if (timer->prev != NULL) { @@ -277,7 +277,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { timerDecRef(timer); removed = true; } - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); return removed; } @@ -372,7 +372,7 @@ static void taosTimerLoopFunc(int32_t signo) { time_wheel_t* wheel = wheels + i; while (now >= wheel->nextScanAt) { - pthread_mutex_lock(&wheel->mutex); + taosThreadMutexLock(&wheel->mutex); wheel->index = (wheel->index + 1) % wheel->size; tmr_obj_t* timer = wheel->slots[wheel->index]; while (timer != NULL) { @@ -407,7 +407,7 @@ static void taosTimerLoopFunc(int32_t signo) { timer = next; } wheel->nextScanAt += wheel->resolution; - pthread_mutex_unlock(&wheel->mutex); + taosThreadMutexUnlock(&wheel->mutex); } addToExpired(expired); @@ -528,12 +528,12 @@ static void taosTmrModuleInit(void) { (tmrCtrls + tsMaxTmrCtrl - 1)->next = NULL; unusedTmrCtrl = tmrCtrls; - pthread_mutex_init(&tmrCtrlMutex, NULL); + taosThreadMutexInit(&tmrCtrlMutex, NULL); int64_t now = taosGetMonotonicMs(); for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; - if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { + if (taosThreadMutexInit(&wheel->mutex, NULL) != 0) { tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno)); return; } @@ -564,15 +564,15 @@ void* taosTmrInit(int32_t maxNumOfTmrs, int32_t resolution, int32_t longest, con const char* ret = taosMonotonicInit(); tmrDebug("ttimer monotonic clock source:%s", ret); - pthread_once(&tmrModuleInit, taosTmrModuleInit); + taosThreadOnce(&tmrModuleInit, taosTmrModuleInit); - pthread_mutex_lock(&tmrCtrlMutex); + taosThreadMutexLock(&tmrCtrlMutex); tmr_ctrl_t* ctrl = unusedTmrCtrl; if (ctrl != NULL) { unusedTmrCtrl = ctrl->next; numOfTmrCtrl++; } - pthread_mutex_unlock(&tmrCtrlMutex); + taosThreadMutexUnlock(&tmrCtrlMutex); if (ctrl == NULL) { tmrError("%s too many timer controllers, failed to create timer controller.", label); @@ -594,11 +594,11 @@ void taosTmrCleanUp(void* handle) { tmrDebug("%s timer controller is cleaned up.", ctrl->label); ctrl->label[0] = 0; - pthread_mutex_lock(&tmrCtrlMutex); + taosThreadMutexLock(&tmrCtrlMutex); ctrl->next = unusedTmrCtrl; numOfTmrCtrl--; unusedTmrCtrl = ctrl; - pthread_mutex_unlock(&tmrCtrlMutex); + taosThreadMutexUnlock(&tmrCtrlMutex); tmrDebug("time controller's tmr ctrl size: %d", numOfTmrCtrl); if (numOfTmrCtrl <= 0) { @@ -608,11 +608,11 @@ void taosTmrCleanUp(void* handle) { for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; - pthread_mutex_destroy(&wheel->mutex); + taosThreadMutexDestroy(&wheel->mutex); free(wheel->slots); } - pthread_mutex_destroy(&tmrCtrlMutex); + taosThreadMutexDestroy(&tmrCtrlMutex); for (size_t i = 0; i < timerMap.size; i++) { timer_list_t* list = timerMap.slots + i; @@ -628,7 +628,7 @@ void taosTmrCleanUp(void* handle) { tmrCtrls = NULL; unusedTmrCtrl = NULL; -#if !defined(WINDOWS) +#if defined(LINUX) tmrModuleInit = PTHREAD_ONCE_INIT; // to support restart #endif } diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c new file mode 100644 index 0000000000000000000000000000000000000000..69cf7baaadf9d1a282a91ba1f2ea4faca363a7ad --- /dev/null +++ b/source/util/src/tuuid.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tuuid.h" + +static int64_t tUUIDHashId = 0; +static int32_t tUUIDSerialNo = 0; + +int32_t tGenIdPI32(void) { + if (tUUIDHashId == 0) { + char uid[64]; + int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + if (code != TSDB_CODE_SUCCESS) { + terrno = TAOS_SYSTEM_ERROR(errno); + } else { + tUUIDHashId = MurmurHash3_32(uid, strlen(uid)); + } + } + + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1); + + int32_t id = ((tUUIDHashId & 0x1F) << 26) | ((pid & 0x3F) << 20) | ((ts & 0xFFF) << 8) | (val & 0xFF); + return id; +} + +int64_t tGenIdPI64(void) { + if (tUUIDHashId == 0) { + char uid[64]; + int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + if (code != TSDB_CODE_SUCCESS) { + terrno = TAOS_SYSTEM_ERROR(errno); + } else { + tUUIDHashId = MurmurHash3_32(uid, strlen(uid)); + } + } + + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1); + + int64_t id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + return id; +} diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 1fa70da870ffd9fe33fb24cfff9b6f3f48115154..78098af3bdc8a5d371bf42f25c5d225bbc17784c 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -28,7 +28,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { return -1; } - if (pthread_mutex_init(&pool->mutex, NULL)) { + if (taosThreadMutexInit(&pool->mutex, NULL)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -56,13 +56,13 @@ void tQWorkerCleanup(SQWorkerPool *pool) { SQWorker *worker = pool->workers + i; if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { - pthread_join(worker->thread, NULL); + taosThreadJoin(worker->thread, NULL); } } tfree(pool->workers); taosCloseQset(pool->qset); - pthread_mutex_destroy(&pool->mutex); + taosThreadMutexDestroy(&pool->mutex); uDebug("worker:%s is closed", pool->name); } @@ -94,10 +94,10 @@ static void *tQWorkerThreadFp(SQWorker *worker) { } STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, ThreadFp threadFp) { - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -110,11 +110,11 @@ STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, Threa do { SQWorker *worker = pool->workers + pool->num; - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, threadFp, worker) != 0) { + if (taosThreadCreate(&worker->thread, &thAttr, threadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosCloseQueue(queue); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -122,13 +122,13 @@ STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, Threa break; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); pool->num++; uDebug("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num); } while (pool->num < pool->min); } - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); return queue; @@ -194,7 +194,7 @@ int32_t tWWorkerInit(SWWorkerPool *pool) { return -1; } - if (pthread_mutex_init(&pool->mutex, NULL) != 0) { + if (taosThreadMutexInit(&pool->mutex, NULL) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -224,14 +224,14 @@ void tWWorkerCleanup(SWWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { - pthread_join(worker->thread, NULL); + taosThreadJoin(worker->thread, NULL); taosFreeQall(worker->qall); taosCloseQset(worker->qset); } } tfree(pool->workers); - pthread_mutex_destroy(&pool->mutex); + taosThreadMutexDestroy(&pool->mutex); uInfo("worker:%s is closed", pool->name); } @@ -265,12 +265,12 @@ static void *tWWorkerThreadFp(SWWorker *worker) { } STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { - pthread_mutex_lock(&pool->mutex); + taosThreadMutexLock(&pool->mutex); SWWorker *worker = pool->workers + pool->nextId; STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -281,7 +281,7 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { worker->qset = taosOpenQset(); if (worker->qset == NULL) { taosCloseQueue(queue); - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); return NULL; } @@ -290,15 +290,15 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { if (worker->qall == NULL) { taosCloseQset(worker->qset); taosCloseQueue(queue); - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { + if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosFreeQall(worker->qall); taosCloseQset(worker->qset); @@ -310,13 +310,13 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pool->nextId = (pool->nextId + 1) % pool->max; } - pthread_attr_destroy(&thAttr); + taosThreadAttrDestroy(&thAttr); } else { taosAddIntoQset(worker->qset, queue, ahandle); pool->nextId = (pool->nextId + 1) % pool->max; } - pthread_mutex_unlock(&pool->mutex); + taosThreadMutexUnlock(&pool->mutex); uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); return queue; diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 25314842fb4d417b29dca127a1da70d55b87bd4f..b4da43dfb72600dbcc6ea5fda1fdde220e99c803 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -230,7 +230,7 @@ static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { const char *tstr; uint64_t len; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; - pSAV1->A_c = (char *)TCODER_MALLOC(len + 1, pCoder); + TCODER_MALLOC(pSAV1->A_c, char*, len + 1, pCoder); memcpy(pSAV1->A_c, tstr, len + 1); tEndDecode(pCoder); @@ -269,7 +269,7 @@ static int32_t tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { const char *tstr; uint64_t len; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; - pSAV2->A_c = (char *)TCODER_MALLOC(len + 1, pCoder); + TCODER_MALLOC(pSAV2->A_c, char*, len + 1, pCoder); memcpy(pSAV2->A_c, tstr, len + 1); // ------------------------NEW FIELDS DECODE------------------------------- @@ -305,7 +305,7 @@ static int32_t tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { static int32_t tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { if (tStartDecode(pCoder) < 0) return -1; - ps1->pA = (SStructA_v1 *)TCODER_MALLOC(sizeof(*(ps1->pA)), pCoder); + TCODER_MALLOC(ps1->pA, SStructA_v1*, sizeof(*(ps1->pA)), pCoder); if (tSStructA_v1_decode(pCoder, ps1->pA) < 0) return -1; if (tDecodeI32(pCoder, &ps1->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps1->v_b) < 0) return -1; @@ -339,7 +339,7 @@ static int32_t tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { if (tStartDecode(pCoder) < 0) return -1; - ps2->pA = (SStructA_v2 *)TCODER_MALLOC(sizeof(*(ps2->pA)), pCoder); + TCODER_MALLOC(ps2->pA, SStructA_v2*, sizeof(*(ps2->pA)), pCoder); if (tSStructA_v2_decode(pCoder, ps2->pA) < 0) return -1; if (tDecodeI32(pCoder, &ps2->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps2->v_b) < 0) return -1; diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index 7fd3d693fb6ccd3cae2a0a916b3c0aaa7dc6d071..440e997042a4b7f3395f32d5b0eccfe2960a5c0f 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -8,7 +8,8 @@ TEST(TD_UTIL_FREELIST_TEST, simple_test) { tFreeListInit(&fl); for (size_t i = 0; i < 1000; i++) { - void *ptr = TFL_MALLOC(1024, &fl); + void *ptr = NULL; + TFL_MALLOC(ptr, void*, 1024, &fl); GTEST_ASSERT_NE(ptr, nullptr); } diff --git a/source/util/test/queueTest.cpp b/source/util/test/queueTest.cpp index 310ae4350e2f354e01fc0cced7e7506fe6df9b26..09c544df9f27f9f897d6376e5c08fbee9efe2057 100644 --- a/source/util/test/queueTest.cpp +++ b/source/util/test/queueTest.cpp @@ -14,6 +14,9 @@ #include "os.h" #include "tqueue.h" +#include +#include + class UtilTestQueue : public ::testing::Test { public: void SetUp() override {} @@ -24,6 +27,107 @@ class UtilTestQueue : public ::testing::Test { static void TearDownTestSuite() {} }; -TEST_F(UtilTestQueue, 01_ReadQitemFromQsetByThread) { - EXPECT_EQ(0, 0); -} \ No newline at end of file +#if 0 +TEST_F(UtilTestQueue, 01_fork) { + pid_t pid; + int shmid; + int* shmptr; + int* tmp; + + int err; + pthread_mutexattr_t mattr; + if ((err = pthread_mutexattr_init(&mattr)) < 0) { + printf("mutex addr init error:%s\n", strerror(err)); + exit(1); + } + + if ((err = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) < 0) { + printf("mutex addr get shared error:%s\n", strerror(err)); + exit(1); + } + + pthread_mutex_t* m; + int mid = shmget(IPC_PRIVATE, sizeof(pthread_mutex_t), 0600); + m = (pthread_mutex_t*)shmat(mid, NULL, 0); + + if ((err = pthread_mutex_init(m, &mattr)) < 0) { + printf("mutex mutex init error:%s\n", strerror(err)); + exit(1); + } + + if ((shmid = shmget(IPC_PRIVATE, 1000, IPC_CREAT | 0600)) < 0) { + perror("shmget error"); + exit(1); + } + + if ((shmptr = (int*)shmat(shmid, 0, 0)) == (void*)-1) { + perror("shmat error"); + exit(1); + } + + tmp = shmptr; + + int shmid2; + int** shmptr2; + if ((shmid2 = shmget(IPC_PRIVATE, 20, IPC_CREAT | 0600)) < 0) { + perror("shmget2 error"); + exit(1); + } + + if ((shmptr2 = (int**)shmat(shmid2, 0, 0)) == (void*)-1) { + perror("shmat2 error"); + exit(1); + } + + *shmptr2 = shmptr; + + if ((pid = fork()) < 0) { + perror("fork error"); + exit(1); + } else if (pid == 0) { + if ((err = taosThreadMutexLock(m)) < 0) { + printf("lock error:%s\n", strerror(err)); + exit(1); + } + for (int i = 0; i < 30; ++i) { + **shmptr2 = i; + (*shmptr2)++; + } + + if ((err = taosThreadMutexUnlock(m)) < 0) { + printf("unlock error:%s\n", strerror(err)); + exit(1); + } + exit(0); + + } else { + if ((err = taosThreadMutexLock(m)) < 0) { + printf("lock error:%s\n", strerror(err)); + exit(1); + } + for (int i = 10; i < 42; ++i) { + **shmptr2 = i; + (*shmptr2)++; + } + if ((err = taosThreadMutexUnlock(m)) < 0) { + printf("unlock error:%s\n", strerror(err)); + exit(1); + } + } + + wait(NULL); + + for (int i = 0; i < 70; ++i) { + printf("%d ", tmp[i]); + } + + printf("\n"); + + taosThreadAttrDestroy(&mattr); + //销毁mutex + pthread_mutex_destroy(m); + + exit(0); +} + +#endif \ No newline at end of file diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 58d9d2202ec66459e50799271332e1d398a72ad6..a439a84562bd00675e1d50620cfdc30875c687aa 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -1,6 +1,5 @@ #include #include -#include #include #include #include "os.h" @@ -102,18 +101,18 @@ void *openRefSpace(void *param) { pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); pSpace->rid = calloc(pSpace->refNum, sizeof(int64_t)); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_t thread1, thread2, thread3; - pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace)); - pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace)); - pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace)); + TdThread thread1, thread2, thread3; + taosThreadCreate(&(thread1), &thattr, addRef, (void *)(pSpace)); + taosThreadCreate(&(thread2), &thattr, removeRef, (void *)(pSpace)); + taosThreadCreate(&(thread3), &thattr, acquireRelease, (void *)(pSpace)); - pthread_join(thread1, NULL); - pthread_join(thread2, NULL); - pthread_join(thread3, NULL); + taosThreadJoin(thread1, NULL); + taosThreadJoin(thread2, NULL); + taosThreadJoin(thread3, NULL); for (int i=0; irefNum; ++i) { taosRemoveRef(pSpace->rsetId, pSpace->rid[i]); @@ -161,22 +160,22 @@ int main(int argc, char *argv[]) { taosInitLog("tref.log", 10); SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads); + TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), threads); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i=0; i #include #include #include @@ -5067,11 +5066,11 @@ int main(int argc, char *argv[]) exit(1); } - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), 4); + TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), 4); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); T_par par[4]; par[0].taos = taos[0]; @@ -5083,10 +5082,10 @@ int main(int argc, char *argv[]) par[3].taos = taos[3]; par[3].idx = 3; - pthread_create(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); - //pthread_create(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); - //pthread_create(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); - //pthread_create(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); + taosThreadCreate(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); + //taosThreadCreate(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); + //taosThreadCreate(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); + //taosThreadCreate(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); while(1) { taosSsleep(1); diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index 57cbdf10900c36f9d4ce155caad4d64cd93b0078..a6c20d282e44f48e8c54045fca9d377fca7d236a 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -1,7 +1,6 @@ // TAOS standard API example. The same syntax as MySQL, but only a subet // to compile: gcc -o prepare prepare.c -ltaos -#include #include #include #include @@ -5080,7 +5079,7 @@ int main(int argc, char *argv[]) #if 0 printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); - pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), (size_t)threadNum); + TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), (size_t)threadNum); ThreadInfo* threadInfo = (ThreadInfo *) calloc(sizeof(ThreadInfo), (size_t)threadNum); ThreadInfo* tInfo = threadInfo; @@ -5094,16 +5093,16 @@ int main(int argc, char *argv[]) tInfo->taos = taos; tInfo->idx = i; if (0 == i) { - //pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); - pthread_create(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); + //taosThreadCreate(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + taosThreadCreate(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); } else if (1 == i){ - pthread_create(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); + taosThreadCreate(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); } tInfo++; } for (int i = 0; i < threadNum; i++) { - pthread_join(pThreadList[i], NULL); + taosThreadJoin(pThreadList[i], NULL); } free(pThreadList); diff --git a/tests/script/api/stmtTest.c b/tests/script/api/stmtTest.c index c69de46f8456b50e575db6b55e9bc9a7fa179543..46f1c7c8f8cbaf9523240e71458df23d78c54382 100644 --- a/tests/script/api/stmtTest.c +++ b/tests/script/api/stmtTest.c @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c index 4521ad4fa300e1ad9caca8cbaddbbbfd376e0045..a5427844d75a532e7c06936a39c84d90cf2cff0b 100644 --- a/tests/script/api/stmt_function.c +++ b/tests/script/api/stmt_function.c @@ -1,5 +1,4 @@ #include -#include #include #include #include diff --git a/tests/script/http/httpTest.c b/tests/script/http/httpTest.c index 36ce6b95ba7836893aeec0831d60a289ff658f66..0fe4265a9da316527d4d95239a2ec01d496b8a27 100644 --- a/tests/script/http/httpTest.c +++ b/tests/script/http/httpTest.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -107,16 +106,16 @@ void multiThread() { ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } free(threads); } diff --git a/tests/script/http/httpTestSqlUtc.c b/tests/script/http/httpTestSqlUtc.c index 643c884a1a64d6eaaeb8984cd80d985408edf0e6..c6c5829b95d10794d65c6e910428c96800109b34 100644 --- a/tests/script/http/httpTestSqlUtc.c +++ b/tests/script/http/httpTestSqlUtc.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -107,16 +106,16 @@ void multiThread() { ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } free(threads); } diff --git a/tests/script/http/httpTestSqlt.c b/tests/script/http/httpTestSqlt.c index 2eaaee0f992d802d57b4fc4d684da4622ea3b763..400428f471887da5f815a09aa875b2290f6398ba 100644 --- a/tests/script/http/httpTestSqlt.c +++ b/tests/script/http/httpTestSqlt.c @@ -5,12 +5,11 @@ #include #include #include -#include #define MAXLINE 1024 typedef struct { - pthread_t pid; + TdThread pid; int threadId; int rows; int tables; @@ -107,16 +106,16 @@ void multiThread() { ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; - pthread_attr_t thattr; + TdThreadAttr thattr; pthread->threadId = i + 1; pthread->rows = numOfRows; pthread->tables = numOfTables; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - pthread_create(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadCreate(&pthread->pid, &thattr, (void *(*)(void *))execute, pthread); } for (int i = 0; i < numOfThreads; i++) { - pthread_join(threads[i].pid, NULL); + taosThreadJoin(threads[i].pid, NULL); } free(threads); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index cafca76761bcf4924b0f997ef5f7de570c147c65..0c2b4fe1f4dad86661e70b06282b3cd8e9db86a9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -18,4 +18,7 @@ # ---- insert ./test.sh -f tsim/insert/basic0.sim + +# ---- query +./test.sh -f tsim/query/interval.sim #======================b1-end=============== diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 05f756ebb6d01b82d55692a9f9aa06585592990b..8d7dba2de2a456a21e051ffa04acecf17157af25 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -74,7 +74,7 @@ BUILD_DIR=$TAOS_DIR/$BIN_DIR SIM_DIR=$TAOS_DIR/sim NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/source/dnode/mgmt/daemon +EXE_DIR=$BUILD_DIR/source/dnode/mgmt/main CFG_DIR=$NODE_DIR/cfg LOG_DIR=$NODE_DIR/log DATA_DIR=$NODE_DIR/data diff --git a/tests/script/tsim/insert/basic0.sim b/tests/script/tsim/insert/basic0.sim index eb4780caac70747ff7745aa5601db9ffe8995854..6c24662be70706db3e0fba36e9d5368a30749bfe 100644 --- a/tests/script/tsim/insert/basic0.sim +++ b/tests/script/tsim/insert/basic0.sim @@ -26,9 +26,10 @@ endi print =============== create child table sql create table ct1 using stb tags(1000) sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) sql show tables -if $rows != 2 then +if $rows != 3 then return -1 endi @@ -45,6 +46,8 @@ sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 1 sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) +sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) + #=================================================================== #=================================================================== print =============== query data from child table diff --git a/tests/script/tsim/insert/basic1.sim b/tests/script/tsim/insert/basic1.sim new file mode 100644 index 0000000000000000000000000000000000000000..3fc635532ac409000988808249d0310b7096c5e7 --- /dev/null +++ b/tests/script/tsim/insert/basic1.sim @@ -0,0 +1,94 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d1 +sql show databases +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use d1 + +print =============== create super table, include all type +sql create table if not exists stb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(16), c9 nchar(16), c10 timestamp, c11 tinyint unsigned, c12 smallint unsigned, c13 int unsigned, c14 bigint unsigned) tags (t1 bool, t2 tinyint, t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 binary(16), t9 nchar(16), t10 timestamp, t11 tinyint unsigned, t12 smallint unsigned, t13 int unsigned, t14 bigint unsigned) + +sql create stable if not exists stb_1 (ts timestamp, i int) tags (j int) +sql create table stb_2 (ts timestamp, i int) tags (j int) +sql create stable stb_3 (ts timestamp, i int) tags (j int) + +sql show stables +if $rows != 4 then + return -1 +endi + +print =============== create child table +sql create table c1 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +sql create table c2 using stb tags(false, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 2', 'child tbl 2', '2022-02-25 18:00:00.000', 10, 20, 30, 40) + +sql show tables +if $rows != 2 then + return -1 +endi + + +print =============== insert data, mode1: one row one table in sql +print =============== insert data, mode1: mulit rows one table in sql +print =============== insert data, mode1: one rows mulit table in sql +print =============== insert data, mode1: mulit rows mulit table in sql +sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) + +print =============== query data +sql select * from c1 +if $rows != 4 then + return -1 +endi + +if $data01 != true then + return -1 +endi + +if $data02 != -1 then + return -1 +endi + +if $data03 != -2 then + return -1 +endi + +print =============== query data from st +sql select * from st +if $rows != 4 then + return -1 +endi + +print =============== stop and restart taosd +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +print =============== query data +sql select * from c1 +if $rows != 4 then + return -1 +endi + +if $data01 != true then + return -1 +endi + +if $data02 != -1 then + return -1 +endi + +if $data03 != -2 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim new file mode 100644 index 0000000000000000000000000000000000000000..08a10f26e4b61d840e3abee9d55ff0d82dcd96ca --- /dev/null +++ b/tests/script/tsim/query/interval-offset.sim @@ -0,0 +1,305 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 500 +sql connect + +print =============== create database +sql drop database d0 -x step1 +step1: +sql create database d0 +sql show databases +if $rows != 1 then + return -1 +endi + +sql use d0 + +print =============== create super table and child table +sql create table stb (ts timestamp, tbcol int) tags (t1 int) +sql show stables +print $rows $data00 $data01 $data02 +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb tags ( 1 ) +sql create table ct2 using stb tags ( 2 ) +sql show tables +print $rows $data00 $data10 $data20 +if $rows != 2 then + return -1 +endi + +print =============== insert data into child table +sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:06.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:10.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:16.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:20.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:26.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:30.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:36.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:40.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:46.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:50.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:01:56.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:02:00.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:02:06.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:02:10.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:02:16.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:02:20.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:02:26.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:02:30.000', 1 ) +sql insert into ct1 values ( '2022-01-01 01:02:36.000', 1 ) + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) +print ===> $rows $data00 $data01 $data02 $data03 $data04 +if $rows != 10 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data04 != 2 then + return -1 +endi + +sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-01 12:00:01.000', 2 ) +sql insert into ct2 values ( '2022-01-01 23:00:01.000', 3 ) +sql insert into ct2 values ( '2022-01-02 10:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-03 10:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-04 10:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-05 10:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-06 10:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-07 10:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-08 10:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-09 10:00:01.000', 1 ) +sql insert into ct2 values ( '2022-01-10 10:00:01.000', 1 ) + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2w) +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +if $rows != 11 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi + +return + +sql select count(*) from car interval(1n, 10d) order by ts desc +# tdSql.checkData(0, 1, 1) +# tdSql.checkData(1, 1, 2) +# tdSql.checkData(2, 1, 3) +# tdSql.checkData(3, 1, 3) +# tdSql.checkData(4, 1, 6) +# tdSql.checkData(5, 1, 1) +# tdSql.checkData(6, 1, 1) +# +sql select count(*) from car interval(2n, 5d) +# tdSql.checkData(0, 1, 1) +# tdSql.checkData(1, 1, 1) +# tdSql.checkData(2, 1, 6) +# tdSql.checkData(3, 1, 6) +# tdSql.checkData(4, 1, 3) + +sql select count(*) from car interval(2n) order by ts desc +# tdSql.checkData(0, 1, 3) +# tdSql.checkData(1, 1, 6) +# tdSql.checkData(2, 1, 6) +# tdSql.checkData(3, 1, 1) +# tdSql.checkData(4, 1, 1) +# +sql select count(*) from car interval(1y, 1n) +# tdSql.checkData(0, 1, 1) +# tdSql.checkData(1, 1, 8) +# tdSql.checkData(2, 1, 8) +# +sql select count(*) from car interval(1y, 2n) +# tdSql.checkData(0, 1, 1) +# tdSql.checkData(1, 1, 11) +# tdSql.checkData(2, 1, 5) + +sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) +# tdSql.checkData(0, 1, 6) +# tdSql.checkData(1, 1, 9) + + + + + + + + + + + + +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +print ====== start create child tables and insert data +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) +print ===> $rows $data01 $data05 +if $rows != $rowNum then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + +#print =============== step3 +#$cc = 4 * 60000 +#$ms = 1601481600000 + $cc +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) +#print ===> $rows $data01 $data05 +#if $rows != 5 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi + +#print =============== step4 +#$cc = 40 * 60000 +#$ms = 1601481600000 + $cc + +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc + +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) +#print ===> $rows $data01 $data05 +#if $rows != 20 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi + +#print =============== step5 +#$cc = 40 * 60000 +#$ms = 1601481600000 + $cc + +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc + +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) +#print ===> $rows $data21 $data25 +#if $rows != 42 then +# return -1 +#endi +#if $data20 != 1 then +# return -1 +#endi +#if $data24 != 1 then +# return -1 +#endi + +#print =============== step6 +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) +#print ===> $rows $data11 +#if $rows != 20 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +#print =============== step7 +#$cc = 4 * 60000 +#$ms = 1601481600000 + $cc +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) +#print ===> $rows $data11 +#if $rows != 5 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +#print =============== step8 +#$cc = 40 * 60000 +#$ms1 = 1601481600000 + $cc +# +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc +# +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) +#print ===> $rows $data11 +#if $rows != 20 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi +# +#print =============== step9 +#$cc = 40 * 60000 +#$ms1 = 1601481600000 + $cc +# +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc +# +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) +#print ===> $rows $data11 +#if $rows != 42 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +print =============== clear +#sql drop database $db +#sql show databases +#if $rows != 0 then +# return -1 +#endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim new file mode 100644 index 0000000000000000000000000000000000000000..6dd0a9537e0de9452f5d13659eb4372faf871e40 --- /dev/null +++ b/tests/script/tsim/query/interval.sim @@ -0,0 +1,183 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wal -v 1 +system sh/exec.sh -n dnode1 -s start +sleep 500 +sql connect + +$dbPrefix = m_in_db +$tbPrefix = m_in_tb +$mtPrefix = m_in_mt +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database $db -x step1 +step1: +sql create database $db +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +print ====== start create child tables and insert data +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) +print ===> $rows $data01 $data05 +if $rows != $rowNum then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + +#print =============== step3 +#$cc = 4 * 60000 +#$ms = 1601481600000 + $cc +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) +#print ===> $rows $data01 $data05 +#if $rows != 5 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi + +#print =============== step4 +#$cc = 40 * 60000 +#$ms = 1601481600000 + $cc + +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc + +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) +#print ===> $rows $data01 $data05 +#if $rows != 20 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi + +#print =============== step5 +#$cc = 40 * 60000 +#$ms = 1601481600000 + $cc + +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc + +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) +#print ===> $rows $data21 $data25 +#if $rows != 42 then +# return -1 +#endi +#if $data20 != 1 then +# return -1 +#endi +#if $data24 != 1 then +# return -1 +#endi + +#print =============== step6 +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) +#print ===> $rows $data11 +#if $rows != 20 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +#print =============== step7 +#$cc = 4 * 60000 +#$ms = 1601481600000 + $cc +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) +#print ===> $rows $data11 +#if $rows != 5 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +#print =============== step8 +#$cc = 40 * 60000 +#$ms1 = 1601481600000 + $cc +# +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc +# +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) +#print ===> $rows $data11 +#if $rows != 20 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi +# +#print =============== step9 +#$cc = 40 * 60000 +#$ms1 = 1601481600000 + $cc +# +#$cc = 1 * 60000 +#$ms2 = 1601481600000 - $cc +# +#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) +#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) +#print ===> $rows $data11 +#if $rows != 42 then +# return -1 +#endi +#if $data11 != 10 then +# return -1 +#endi + +print =============== clear +#sql drop database $db +#sql show databases +#if $rows != 0 then +# return -1 +#endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index 09e6ede77dcdc65f8eb6c13c1d26e4eea33006a6..257c26415994fa245fa42e285c667598d6615d1c 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -253,7 +253,8 @@ endi print =============== query data from normal table after restart dnode sql use ndb sql select * from nt1 -if $rows != 3 then +if $rows != 3 then + print expect 3, actual: $rows return -1 endi diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 57fbd4c9c1a55707f66f813fc17272e9526ffab0..5e539780f46854b9ab85c0ba033c9b0923203a5e 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -49,7 +49,7 @@ typedef struct { int64_t startMs; int64_t maxDelay; int64_t minDelay; - pthread_t thread; + TdThread thread; } SThreadInfo; // void parseArgument(int32_t argc, char *argv[]); @@ -400,9 +400,9 @@ int32_t main(int32_t argc, char *argv[]) { pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset); - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); // int64_t numOfTablesPerThread = numOfTables / numOfThreads; @@ -430,12 +430,12 @@ int32_t main(int32_t argc, char *argv[]) { pInfo[i].minDelay = INT64_MAX; strcpy(pInfo[i].dbName, dbName); strcpy(pInfo[i].stbName, stbName); - pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); } taosMsleep(300); for (int32_t i = 0; i < numOfThreads; i++) { - pthread_join(pInfo[i].thread, NULL); + taosThreadJoin(pInfo[i].thread, NULL); } int64_t maxDelay = 0; @@ -465,6 +465,6 @@ int32_t main(int32_t argc, char *argv[]) { numOfThreads, NC); } - pthread_attr_destroy(&thattr); + taosThreadAttrDestroy(&thattr); free(pInfo); } diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h index 24619a4ad59fdafc8c3bf7e13d7b01ca6ba5e02a..1e2190e3086af93e0e1dccd4bc1fa02a65c48635 100644 --- a/tests/tsim/inc/simInt.h +++ b/tests/tsim/inc/simInt.h @@ -143,7 +143,7 @@ typedef struct _script_t { char *optionBuffer; SCmdLine *lines; // command list SVariable variables[MAX_VAR_LEN]; - pthread_t bgPid; + TdThread bgPid; char auth[128]; struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; } SScript; diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 1857e217531a0fcef9f81b8dd258f1e3a4f9a3eb..855705e904060f1646799b54d0f670da2fb36dd7 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -293,7 +293,7 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { script->bgScripts[script->bgScriptLen++] = newScript; simInfo("script:%s, start to execute in background,", newScript->fileName); - if (pthread_create(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { + if (taosThreadCreate(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); return false; } else { diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index bfbfe2b0df8f0d1d680f593f400e4d78df268e35..eddcd3afc226c934c7a310ca5cd9b2bab3a3544f 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -55,7 +55,7 @@ void simFreeScript(SScript *script) { simDebug("script:%s, is background script, set stop flag", bgScript->fileName); bgScript->killed = true; if (taosCheckPthreadValid(bgScript->bgPid)) { - pthread_join(bgScript->bgPid, NULL); + taosThreadJoin(bgScript->bgPid, NULL); } simDebug("script:%s, background thread joined", bgScript->fileName); diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 33d25b67464e52cb6808f6b845963cf65ee49aac..919cb0d84636dec2579b787288820fe2e803c319 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -31,7 +31,7 @@ static int32_t checkedNum = 0; static int32_t errorNum = 0; typedef struct { - pthread_t threadID; + TdThread threadID; int threadIndex; int totalThreads; void * taos; @@ -152,7 +152,7 @@ static void *shellCheckThreadFp(void *arg) { } static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { - pthread_attr_t thattr; + TdThreadAttr thattr; ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; @@ -161,17 +161,17 @@ static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { pThread->taos = con; pThread->db = _args->database; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { + if (taosThreadCreate(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); exit(0); } } for (int t = 0; t < _args->threadNum; ++t) { - pthread_join(threadObj[t].threadID, NULL); + taosThreadJoin(threadObj[t].threadID, NULL); } for (int t = 0; t < _args->threadNum; ++t) { diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index e4cf09358bc1b0b6e4777577373288c577477efc..69c7a7bc4e568f9f63a0b63be912f41be7adb8bb 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -28,7 +28,6 @@ int indicator = 1; struct termios oldtio; -extern int wcwidth(wchar_t c); void insertChar(Command *cmd, char *c, int size); @@ -366,7 +365,7 @@ void *shellLoopQuery(void *arg) { setThreadName("shellLoopQuery"); - pthread_cleanup_push(cleanup_handler, NULL); + taosThreadCleanupPush(cleanup_handler, NULL); char *command = malloc(MAX_COMMAND_SIZE); if (command == NULL){ @@ -390,7 +389,7 @@ void *shellLoopQuery(void *arg) { tfree(command); exitShell(); - pthread_cleanup_pop(1); + taosThreadCleanupPop(1); return NULL; } @@ -426,7 +425,7 @@ void showOnScreen(Command *cmd) { w.ws_row = 30; } - wchar_t wc; + TdWchar wc; int size = 0; // Print out the command. @@ -441,11 +440,11 @@ void showOnScreen(Command *cmd) { int remain_column = w.ws_col; /* size = cmd->commandSize + prompt_size; */ for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = mbtowc(&wc, str, MB_CUR_MAX); + int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); if (ret < 0) break; size += ret; /* assert(size >= 0); */ - int width = wcwidth(wc); + int width = taosWcharWidth(wc); if (remain_column > width) { printf("%lc", wc); remain_column -= width; diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index b42f77e87e0a90e152d1a8b937ab8a27e6a5ec20..398bcf2280f5946bf1c2d5424813c8a507ed9a5b 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -28,7 +28,7 @@ static int32_t shellSQLFileNum = 0; static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0}; typedef struct { - pthread_t threadID; + TdThread threadID; int threadIndex; int totalThreads; void *taos; @@ -232,7 +232,7 @@ void* shellImportThreadFp(void *arg) static void shellRunImportThreads(SShellArguments* _args) { - pthread_attr_t thattr; + TdThreadAttr thattr; ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; @@ -244,17 +244,17 @@ static void shellRunImportThreads(SShellArguments* _args) exit(0); } - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) { + if (taosThreadCreate(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) { fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); exit(0); } } for (int t = 0; t < _args->threadNum; ++t) { - pthread_join(threadObj[t].threadID, NULL); + taosThreadJoin(threadObj[t].threadID, NULL); } for (int t = 0; t < _args->threadNum; ++t) { diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index 772d92d8c6956a3eed1d5273df984c15b73cffc8..ee32bfb6be47ab6c6a841c081b3b2eb8d3dd1547 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -519,7 +519,7 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { int32_t num = endPort - startPort + 1; if (num < 0) num = 1; - pthread_t *pids = malloc(2 * num * sizeof(pthread_t)); + TdThread *pids = malloc(2 * num * sizeof(TdThread)); STestInfo *tinfos = malloc(num * sizeof(STestInfo)); STestInfo *uinfos = malloc(num * sizeof(STestInfo)); @@ -528,7 +528,7 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { tcpInfo->port = port + i; tcpInfo->pktLen = pkgLen; - if (pthread_create(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { + if (taosThreadCreate(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } @@ -536,15 +536,15 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { STestInfo *udpInfo = uinfos + i; udpInfo->port = port + i; tcpInfo->pktLen = pkgLen; - if (pthread_create(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { + if (taosThreadCreate(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } } for (int32_t i = 0; i < num; i++) { - pthread_join(pids[i], NULL); - pthread_join(pids[(num + i)], NULL); + taosThreadJoin(pids[i], NULL); + taosThreadJoin(pids[(num + i)], NULL); } } diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index cf0ceded38307e5bd138a77c19203e882f0662e4..fd993998b823411c666194d9f2bb9c6390baa2c4 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -21,8 +21,6 @@ #include -extern int wcwidth(wchar_t c); -extern int wcswidth(const wchar_t *s, size_t n); typedef struct { char widthInString; char widthOnScreen; @@ -43,7 +41,7 @@ int countPrefixOnes(unsigned char c) { void getPrevCharSize(const char *str, int pos, int *size, int *width) { assert(pos > 0); - wchar_t wc; + TdWchar wc; *size = 0; *width = 0; @@ -53,25 +51,25 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) { if (str[pos] > 0 || countPrefixOnes((unsigned char )str[pos]) > 1) break; } - int rc = mbtowc(&wc, str + pos, MB_CUR_MAX); + int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); assert(rc == *size); - *width = wcwidth(wc); + *width = taosWcharWidth(wc); } void getNextCharSize(const char *str, int pos, int *size, int *width) { assert(pos >= 0); - wchar_t wc; - *size = mbtowc(&wc, str + pos, MB_CUR_MAX); - *width = wcwidth(wc); + TdWchar wc; + *size = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + *width = taosWcharWidth(wc); } void insertChar(Command *cmd, char *c, int size) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - wchar_t wc; - if (mbtowc(&wc, c, size) < 0) return; + TdWchar wc; + if (taosMbToWchar(&wc, c, size) < 0) return; clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); /* update the buffer */ @@ -81,8 +79,8 @@ void insertChar(Command *cmd, char *c, int size) { /* update the values */ cmd->commandSize += size; cmd->cursorOffset += size; - cmd->screenOffset += wcwidth(wc); - cmd->endOffset += wcwidth(wc); + cmd->screenOffset += taosWcharWidth(wc); + cmd->endOffset += taosWcharWidth(wc); showOnScreen(cmd); } @@ -249,10 +247,10 @@ int isReadyGo(Command *cmd) { } void getMbSizeInfo(const char *str, int *size, int *width) { - wchar_t *wc = (wchar_t *)calloc(sizeof(wchar_t), MAX_COMMAND_SIZE); + TdWchar *wc = (TdWchar *)calloc(sizeof(TdWchar), MAX_COMMAND_SIZE); *size = strlen(str); - mbstowcs(wc, str, MAX_COMMAND_SIZE); - *width = wcswidth(wc, MAX_COMMAND_SIZE); + taosMbsToWchars(wc, str, MAX_COMMAND_SIZE); + *width = taosWcharsWidth(wc, MAX_COMMAND_SIZE); free(wc); } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 5a989937d8cfd41050bca0a077947a6feda9514b..1b35afb57d231c07181499fa43c66cbad137e72c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -560,12 +560,12 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { } static void shellPrintNChar(const char *str, int length, int width) { - wchar_t tail[3]; + TdWchar tail[3]; int pos = 0, cols = 0, totalCols = 0, tailLen = 0; while (pos < length) { - wchar_t wc; - int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); + TdWchar wc; + int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); if (bytes == 0) { break; } @@ -577,7 +577,7 @@ static void shellPrintNChar(const char *str, int length, int width) { #ifdef WINDOWS int w = bytes; #else - int w = wcwidth(wc); + int w = taosWcharWidth(wc); #endif if (w <= 0) { continue; diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index cc497688d1bfcea14aba2e6869984962b305b075..de5db0b2884b2c3fb9034b015829d59c387b8d75 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -31,7 +31,6 @@ int indicator = 1; struct termios oldtio; -extern int wcwidth(wchar_t c); void insertChar(Command *cmd, char *c, int size); const char *argp_program_version = version; const char *argp_program_bug_address = ""; @@ -396,7 +395,7 @@ void *shellLoopQuery(void *arg) { setThreadName("shellLoopQuery"); - pthread_cleanup_push(cleanup_handler, NULL); + taosThreadCleanupPush(cleanup_handler, NULL); char *command = malloc(MAX_COMMAND_SIZE); if (command == NULL){ @@ -420,7 +419,7 @@ void *shellLoopQuery(void *arg) { tfree(command); exitShell(); - pthread_cleanup_pop(1); + taosThreadCleanupPop(1); return NULL; } @@ -456,7 +455,7 @@ void showOnScreen(Command *cmd) { w.ws_row = 30; } - wchar_t wc; + TdWchar wc; int size = 0; // Print out the command. @@ -471,11 +470,11 @@ void showOnScreen(Command *cmd) { int remain_column = w.ws_col; /* size = cmd->commandSize + prompt_size; */ for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = mbtowc(&wc, str, MB_CUR_MAX); + int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); if (ret < 0) break; size += ret; /* assert(size >= 0); */ - int width = wcwidth(wc); + int width = taosWcharWidth(wc); if (remain_column > width) { printf("%lc", wc); remain_column -= width; diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 2832855517e2529d511a39adcf749b87a6c7d1d3..574e3fa8b840255e04a56f811be270abbbade454 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -17,7 +17,7 @@ #include "shell.h" #include "tglobal.h" -pthread_t pid; +TdThread pid; static tsem_t cancelSem; void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { @@ -140,8 +140,8 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } - pthread_t spid; - pthread_create(&spid, NULL, cancelHandler, NULL); + TdThread spid; + taosThreadCreate(&spid, NULL, cancelHandler, NULL); /* Interrupt handler. */ taosSetSignal(SIGTERM, shellQueryInterruptHandler); @@ -154,7 +154,7 @@ int main(int argc, char *argv[]) { /* Loop to query the input. */ while (1) { - pthread_create(&pid, NULL, shellLoopQuery, con); - pthread_join(pid, NULL); + taosThreadCreate(&pid, NULL, shellLoopQuery, con); + taosThreadJoin(pid, NULL); } }