diff --git a/Jenkinsfile2 b/Jenkinsfile2 index a3006c4f7d0dbbb1282ab6eaba4c66862f7723b0..cbf663cdcfeb1ed1f31adbf152dc2e3ff4fc8b66 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -33,7 +33,18 @@ def abort_previous(){ milestone(buildNumber) } def pre_test(){ - sh 'hostname' + sh ''' + hostname + date + ''' + sh ''' + cd ${WK} + git reset --hard + git fetch || git fetch + cd ${WKC} + git reset --hard + git fetch || git fetch + ''' script { if (env.CHANGE_TARGET == 'master') { sh ''' @@ -222,7 +233,7 @@ pipeline { changeRequest() } steps { - timeout(time: 20, unit: 'MINUTES'){ + timeout(time: 40, unit: 'MINUTES'){ pre_test() script { sh ''' @@ -234,8 +245,9 @@ pipeline { ''' sh ''' cd ${WKC}/tests/parallel_test + export DEFAULT_RETRY_TIME=2 date - time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${CHANGE_TARGET} -l ${WKDIR}/log + timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME} -l ${WKDIR}/log -o 480 ''' } } diff --git a/cmake/cmake.install b/cmake/cmake.install index 5a05c0c7dc8eebf34dc1e5825a8d955462559381..f51c6566fa1058daf3454ef09452981f09584ce4 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -1,5 +1,5 @@ IF (TD_LINUX) - SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/make_install.sh") + SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})") ELSEIF (TD_WINDOWS) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 926fbc8957590511b476ed163ac3df3430cceb6d..14a85ee4f6f028b3af52f8e78c0029fc34a1223a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -365,7 +365,7 @@ if(${BUILD_ADDR2LINE}) if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H) target_link_libraries(libdwarf PUBLIC libelf) endif() - target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_BINARY_DIR}/contrib) + target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR}) file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT) string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 80125d67880043759e86fddb70483480edd7cecf..9e3ad42a82fe779bc507417d84718b342a98a34e 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -201,18 +201,17 @@ typedef struct SExprInfo { } SExprInfo; typedef struct { - const char* key; - int32_t keyLen; - uint8_t type; - int16_t length; + const char* key; + int32_t keyLen; + uint8_t type; union{ const char* value; - int64_t i; - uint64_t u; - double d; - float f; + int64_t i; + uint64_t u; + double d; + float f; }; - int32_t valueLen; + int32_t length; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 84aae46347292fa5b4ea66cf0c435e99494beba2..83283cfdad048a932f9df5da0ecf82d4aeeb1280 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -34,7 +34,6 @@ extern int32_t tsVersion; extern int32_t tsStatusInterval; // common -extern int32_t tsMaxConnections; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; extern int32_t tsCompressMsgSize; @@ -52,6 +51,7 @@ extern int32_t tsVnodeShmSize; extern int32_t tsQnodeShmSize; extern int32_t tsSnodeShmSize; extern int32_t tsBnodeShmSize; +extern int32_t tsNumOfShmThreads; // queue & threads extern int32_t tsNumOfRpcThreads; @@ -68,6 +68,7 @@ extern int32_t tsNumOfQnodeQueryThreads; extern int32_t tsNumOfQnodeFetchThreads; extern int32_t tsNumOfSnodeSharedThreads; extern int32_t tsNumOfSnodeUniqueThreads; +extern int64_t tsRpcQueueMemoryAllowed; // monitor extern bool tsEnableMonitor; diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index b6c96bb2d16e7b3b4bd644aa30a6e825f8c98250..a484c2acc98c969aed77f3cfd9151dd93148d4ea 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -37,11 +37,12 @@ typedef enum { QUEUE_MAX, } EQueueType; -typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); -typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype); +typedef int32_t (*PutToQueueFp)(void *pMgmt, SRpcMsg* pReq); +typedef int32_t (*GetQueueSizeFp)(void *pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq); typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp); +typedef void (*SendMnodeRecvFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq, SRpcMsg* pRsp); typedef void (*SendRedirectRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp, const SEpSet* pNewEpSet); typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type); @@ -49,23 +50,26 @@ typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const ch typedef struct { SMgmtWrapper* pWrapper; + void* pMgmt; + void* clientRpc; PutToQueueFp queueFps[QUEUE_MAX]; GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; SendRspFp sendRspFp; + SendMnodeRecvFp sendMnodeRecvFp; SendRedirectRspFp sendRedirectRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; - void* clientRpc; } SMsgCb; void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq); int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq); -void tmsgSendRsp(const SRpcMsg* pRsp); -void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet); +void tmsgSendRsp(SRpcMsg* pRsp); +void tmsgSendMnodeRecv(SRpcMsg* pReq, SRpcMsg* pRsp); +void tmsgSendRedirectRsp(SRpcMsg* pRsp, const SEpSet* pNewEpSet); void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg); void tmsgReleaseHandle(void* handle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); diff --git a/include/os/osString.h b/include/os/osString.h index 026cb33ad9ddf2e8f18ac84bd8f2779c81701c57..5f65f97bec05b37ae9e901cab51f342c96f5bbd0 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -37,6 +37,7 @@ typedef int32_t TdUcs4; #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID #define wchar_t WCHAR_T_TYPE_TAOS_FORBID + #define strcasestr STR_CASE_STR_FORBID #endif #ifdef WINDOWS @@ -69,6 +70,8 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size); int32_t taosWcharToMb(char *pStr, TdWchar wchar); int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size); +char *taosStrCaseStr(const char *str, const char *pattern); + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ae083780e4f35da4687fecb6fa35744c1f5f5a13..ed541dbb8374087ee5288868c679d9ea9079a791 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -89,6 +89,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0115) #define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0116) #define TSDB_CODE_NEED_RETRY TAOS_DEF_ERROR_CODE(0, 0x0117) +#define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE TAOS_DEF_ERROR_CODE(0, 0x0118) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0140) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0141) diff --git a/include/util/tdef.h b/include/util/tdef.h index 022fd8ba8e517da21a93eccd6b0e1b0e0c617982..f95d96be56d40a24cb227820058807eac9e7f051 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -425,9 +425,12 @@ enum { SND_WORKER_TYPE__UNIQUE, }; -#define MNODE_HANDLE -1 -#define QNODE_HANDLE 1 #define DEFAULT_HANDLE 0 +#define MNODE_HANDLE -1 +#define QNODE_HANDLE -2 +#define SNODE_HANDLE -3 +#define VNODE_HANDLE -4 +#define BNODE_HANDLE -5 #define TSDB_CONFIG_OPTION_LEN 16 #define TSDB_CONIIG_VALUE_LEN 48 diff --git a/include/util/tjson.h b/include/util/tjson.h index d23f7b402ea142ea9118f57f6274983d78c5a72a..a95efe56e7d777be60f51f4424c3619967c14c5a 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -22,17 +22,12 @@ extern "C" { #endif -#ifdef WINDOWS -#define tjsonGetNumberValue(pJson, pName, val) -1 -#else -#define tjsonGetNumberValue(pJson, pName, val) \ - ({ \ - uint64_t _tmp = 0; \ - int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \ - val = _tmp; \ - _code; \ - }) -#endif +#define tjsonGetNumberValue(pJson, pName, val, code) \ + do { \ + uint64_t _tmp = 0; \ + code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \ + val = _tmp; \ + } while (0) typedef void SJson; diff --git a/include/util/tprocess.h b/include/util/tprocess.h index 7e1767441e4e14b0517b53e949ec044d2bd2568e..5e5a982ec4243e09318d858b46596c2d93106685 100644 --- a/include/util/tprocess.h +++ b/include/util/tprocess.h @@ -17,6 +17,7 @@ #define _TD_UTIL_PROCESS_H_ #include "os.h" +#include "tqueue.h" #ifdef __cplusplus extern "C" { @@ -25,7 +26,7 @@ extern "C" { typedef enum { PROC_FUNC_REQ = 1, PROC_FUNC_RSP, PROC_FUNC_REGIST, PROC_FUNC_RELEASE } EProcFuncType; typedef struct SProcObj SProcObj; -typedef void *(*ProcMallocFp)(int32_t contLen); +typedef void *(*ProcMallocFp)(int32_t contLen, EQItype itype); typedef void *(*ProcFreeFp)(void *pCont); typedef void (*ProcConsumeFp)(void *parent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen, EProcFuncType ftype); diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 70db65d50f850e1099032ddbde3e68913f1f5386..dbc4d03177e4c489240c04aac37710ce995102d4 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -48,18 +48,24 @@ typedef struct { int32_t threadNum; } SQueueInfo; +typedef enum { + DEF_QITEM = 0, + RPC_QITEM = 1, +} EQItype; + typedef void (*FItem)(SQueueInfo *pInfo, void *pItem); typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems); STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); -void *taosAllocateQitem(int32_t size); +void *taosAllocateQitem(int32_t size, EQItype itype); void taosFreeQitem(void *pItem); void taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem); bool taosQueueEmpty(STaosQueue *queue); -int32_t taosQueueSize(STaosQueue *queue); +int32_t taosQueueItemSize(STaosQueue *queue); +int64_t taosQueueMemorySize(STaosQueue *queue); STaosQall *taosAllocateQall(); void taosFreeQall(STaosQall *qall); @@ -77,8 +83,8 @@ int32_t taosGetQueueNumber(STaosQset *qset); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp); void taosResetQsetThread(STaosQset *qset, void *pItem); -int32_t taosGetQueueItemsNumber(STaosQueue *queue); -int32_t taosGetQsetItemsNumber(STaosQset *qset); + +extern int64_t tsRpcQueueMemoryAllowed; #ifdef __cplusplus } diff --git a/packaging/cfg/nginxd.service b/packaging/cfg/nginxd.service new file mode 100644 index 0000000000000000000000000000000000000000..97c0e2f82934474946fe3af1e4d9a776c1de21ac --- /dev/null +++ b/packaging/cfg/nginxd.service @@ -0,0 +1,22 @@ +[Unit] +Description=Nginx For TDengine Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=forking +PIDFile=/usr/local/nginxd/logs/nginx.pid +ExecStart=/usr/local/nginxd/sbin/nginx +ExecStop=/usr/local/nginxd/sbin/nginx -s stop +TimeoutStopSec=1000000s +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TimeoutStartSec=0 +StandardOutput=null +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg new file mode 100644 index 0000000000000000000000000000000000000000..4570f2d1240d7953c28459f3d21284c9e3e4611b --- /dev/null +++ b/packaging/cfg/taos.cfg @@ -0,0 +1,312 @@ +######################################################## +# # +# TDengine Configuration # +# Any questions, please email support@taosdata.com # +# # +######################################################## + +# first fully qualified domain name (FQDN) for TDengine system +# firstEp hostname:6030 + +# local fully qualified domain name (FQDN) +# fqdn hostname + +# first port number for the connection (12 continuous UDP/TCP port number are used) +# serverPort 6030 + +# log file's directory +# logDir /var/log/taos + +# data file's directory +# dataDir /var/lib/taos + +# temporary file's directory +# tempDir /tmp/ + +# the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only +# arbitrator arbitrator_hostname:6042 + +# number of threads per CPU core +# numOfThreadsPerCore 1.0 + +# number of threads to commit cache data +# numOfCommitThreads 4 + +# the proportion of total CPU cores available for query processing +# 2.0: the query threads will be set to double of the CPU cores. +# 1.0: all CPU cores are available for query processing [default]. +# 0.5: only half of the CPU cores are available for query. +# 0.0: only one core available. +# ratioOfQueryCores 1.0 + +# the last_row/first/last aggregator will not change the original column name in the result fields +keepColumnName 1 + +# number of management nodes in the system +# numOfMnodes 1 + +# enable/disable backuping vnode directory when removing vnode +# vnodeBak 1 + +# enable/disable installation / usage report +# telemetryReporting 1 + +# enable/disable load balancing +# balance 1 + +# role for dnode. 0 - any, 1 - mnode, 2 - dnode +# role 0 + +# max timer control blocks +# maxTmrCtrl 512 + +# time interval of system monitor, seconds +# monitorInterval 30 + +# number of seconds allowed for a dnode to be offline, for cluster only +# offlineThreshold 864000 + +# RPC re-try timer, millisecond +# rpcTimer 300 + +# RPC maximum time for ack, seconds. +# rpcMaxTime 600 + +# time interval of dnode status reporting to mnode, seconds, for cluster only +# statusInterval 1 + +# time interval of heart beat from shell to dnode, seconds +# shellActivityTimer 3 + +# minimum sliding window time, milli-second +# minSlidingTime 10 + +# minimum time window, milli-second +# minIntervalTime 10 + +# maximum delay before launching a stream computation, milli-second +# maxStreamCompDelay 20000 + +# maximum delay before launching a stream computation for the first time, milli-second +# maxFirstStreamCompDelay 10000 + +# retry delay when a stream computation fails, milli-second +# retryStreamCompDelay 10 + +# the delayed time for launching a stream computation, from 0.1(default, 10% of whole computing time window) to 0.9 +# streamCompDelayRatio 0.1 + +# max number of vgroups per db, 0 means configured automatically +# maxVgroupsPerDb 0 + +# max number of tables per vnode +# maxTablesPerVnode 1000000 + +# cache block size (Mbyte) +# cache 16 + +# number of cache blocks per vnode +# blocks 6 + +# number of days per DB file +# days 10 + +# number of days to keep DB file +# keep 3650 + +# minimum rows of records in file block +# minRows 100 + +# maximum rows of records in file block +# maxRows 4096 + +# the number of acknowledgments required for successful data writing +# quorum 1 + +# enable/disable compression +# comp 2 + +# write ahead log (WAL) level, 0: no wal; 1: write wal, but no fysnc; 2: write wal, and call fsync +# walLevel 1 + +# if walLevel is set to 2, the cycle of fsync being executed, if set to 0, fsync is called right away +# fsync 3000 + +# number of replications, for cluster only +# replica 1 + +# the compressed rpc message, option: +# -1 (no compression) +# 0 (all message compressed), +# > 0 (rpc message body which larger than this value will be compressed) +# compressMsgSize -1 + +# query retrieved column data compression option: +# -1 (no compression) +# 0 (all retrieved column data compressed), +# > 0 (any retrieved column size greater than this value all data will be compressed.) +# compressColData -1 + +# max length of an SQL +# maxSQLLength 65480 + +# max length of WildCards +# maxWildCardsLength 100 + +# the maximum number of records allowed for super table time sorting +# maxNumOfOrderedRes 100000 + +# system time zone +# timezone Asia/Shanghai (CST, +0800) +# system time zone (for windows 10) +# timezone UTC-8 + +# system locale +# locale en_US.UTF-8 + +# default system charset +# charset UTF-8 + +# max number of connections allowed in dnode +# maxShellConns 5000 + +# max number of connections allowed in client +# maxConnections 5000 + +# stop writing logs when the disk size of the log folder is less than this value +# minimalLogDirGB 1.0 + +# stop writing temporary files when the disk size of the tmp folder is less than this value +# minimalTmpDirGB 1.0 + +# if disk free space is less than this value, taosd service exit directly within startup process +# minimalDataDirGB 2.0 + +# One mnode is equal to the number of vnode consumed +# mnodeEqualVnodeNum 4 + +# enbale/disable http service +# http 1 + +# enable/disable system monitor +# monitor 1 + +# enable/disable recording the SQL statements via restful interface +# httpEnableRecordSql 0 + +# number of threads used to process http requests +# httpMaxThreads 2 + +# maximum number of rows returned by the restful interface +# restfulRowLimit 10240 + +# database name must be specified in restful interface if the following parameter is set, off by default +# httpDbNameMandatory 1 + +# http keep alive, default is 30 seconds +# httpKeepAlive 30000 + +# The following parameter is used to limit the maximum number of lines in log files. +# max number of lines per log filters +# numOfLogLines 10000000 + +# enable/disable async log +# asyncLog 1 + +# time of keeping log files, days +# logKeepDays 0 + + +# The following parameters are used for debug purpose only. +# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR +# 131: output warning and error +# 135: output debug, warning and error +# 143: output trace, debug, warning and error to log +# 199: output debug, warning and error to both screen and file +# 207: output trace, debug, warning and error to both screen and file + +# debug flag for all log type, take effect when non-zero value +# debugFlag 0 + +# debug flag for meta management messages +# mDebugFlag 135 + +# debug flag for dnode messages +# dDebugFlag 135 + +# debug flag for sync module +# sDebugFlag 135 + +# debug flag for WAL +# wDebugFlag 135 + +# debug flag for SDB +# sdbDebugFlag 135 + +# debug flag for RPC +# rpcDebugFlag 131 + +# debug flag for TAOS TIMER +# tmrDebugFlag 131 + +# debug flag for TDengine client +# cDebugFlag 131 + +# debug flag for JNI +# jniDebugFlag 131 + +# debug flag for storage +# uDebugFlag 131 + +# debug flag for http server +# httpDebugFlag 131 + +# debug flag for monitor +# monDebugFlag 131 + +# debug flag for query +# qDebugFlag 131 + +# debug flag for vnode +# vDebugFlag 131 + +# debug flag for TSDB +# tsdbDebugFlag 131 + +# debug flag for continue query +# cqDebugFlag 131 + +# enable/disable recording the SQL in taos client +# enableRecordSql 0 + +# generate core file when service crash +# enableCoreFile 1 + +# maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden +# maxBinaryDisplayWidth 30 + +# enable/disable stream (continuous query) +# stream 1 + +# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode +# retrieveBlockingModel 0 + +# the maximum allowed query buffer size in MB during query processing for each data node +# -1 no limit (default) +# 0 no query allowed, queries are disabled +# queryBufferSize -1 + +# percent of redundant data in tsdb meta will compact meta data,0 means donot compact +# tsdbMetaCompactRatio 0 + +# default string type used for storing JSON String, options can be binary/nchar, default is nchar +# defaultJSONStrType nchar + +# force TCP transmission +# rpcForceTcp 0 + +# unit MB. Flush vnode wal file if walSize > walFlushSize and walSize > cache*0.5*blocks +# walFlushSize 1024 + +# unit Hour. Latency of data migration +# keepTimeOffset 0 diff --git a/packaging/cfg/taosd.service b/packaging/cfg/taosd.service new file mode 100644 index 0000000000000000000000000000000000000000..db08001df9a78105d367b96cf62f6c7e8721b34d --- /dev/null +++ b/packaging/cfg/taosd.service @@ -0,0 +1,21 @@ +[Unit] +Description=TDengine server service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/taosd +ExecStartPre=/usr/local/taos/bin/startPre.sh +TimeoutStopSec=1000000s +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TimeoutStartSec=0 +StandardOutput=null +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target diff --git a/packaging/cfg/tarbitratord.service b/packaging/cfg/tarbitratord.service new file mode 100644 index 0000000000000000000000000000000000000000..051b1ae44fb5e9b885e2f4ddaa399b40bd9ac7dd --- /dev/null +++ b/packaging/cfg/tarbitratord.service @@ -0,0 +1,20 @@ +[Unit] +Description=TDengine arbitrator service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/tarbitrator +TimeoutStopSec=1000000s +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TimeoutStartSec=0 +StandardOutput=null +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target diff --git a/packaging/check_package.sh b/packaging/check_package.sh new file mode 100644 index 0000000000000000000000000000000000000000..7b6165016292996d8ccda416aedbadbc961fccd7 --- /dev/null +++ b/packaging/check_package.sh @@ -0,0 +1,252 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +verMode=edge +pagMode=full + +iplist="" +serverFqdn="" + +# -----------------------Variables definition--------------------- +script_dir="../release" +# Dynamic directory +data_dir="/var/lib/taos" +log_dir="/var/log/taos" + +data_link_dir="/usr/local/taos/data" +log_link_dir="/usr/local/taos/log" + +cfg_install_dir="/etc/taos" + +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/taos" + +# old bin dir +sbin_dir="/usr/local/taos/bin" + +temp_version="" +fin_result="" + +service_config_dir="/etc/systemd/system" +nginx_port=6060 +nginx_dir="/usr/local/nginxd" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +# ============================= get input parameters ================================================= + +# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] + +# set parameters by default value +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] + +while getopts "hv:d:" arg +do + case $arg in + d) + #echo "interactiveFqdn=$OPTARG" + script_dir=$( echo $OPTARG ) + ;; + h) + echo "Usage: `basename $0` -d scripy_path" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function check_file() { + #check file whether exists + if [ ! -e $1/$2 ];then + echo -e "$1/$2 \033[31mnot exists\033[0m!quit" + fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n" + echo -e $fin_result + exit 8 + fi +} + +function get_package_name() { + var=$1 + if [[ $1 =~ 'aarch' ]];then + echo ${var::-21} + else + echo ${var::-17} + fi +} + +function check_link() { + #check Link whether exists or broken + if [ -L $1 ] ; then + if [ ! -e $1 ] ; then + echo -e "$1 \033[31Broken link\033[0m" + fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n" + echo -e $fin_result + exit 8 + fi + else + echo -e "$1 \033[31mnot exists\033[0m!quit" + fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n" + echo -e $fin_result + exit 8 + fi +} + +function check_main_path() { + #check install main dir and all sub dir + main_dir=("" "cfg" "bin" "connector" "driver" "examples" "include" "init.d") + for i in "${main_dir[@]}";do + check_file ${install_main_dir} $i + done + if [ "$verMode" == "cluster" ]; then + nginx_main_dir=("admin" "conf" "html" "sbin" "logs") + for i in "${nginx_main_dir[@]}";do + check_file ${nginx_dir} $i + done + fi + echo -e "Check main path:\033[32mOK\033[0m!" +} + +function check_bin_path() { + # check install bin dir and all sub dir + bin_dir=("taos" "taosd" "taosadapter" "taosdemo" "remove.sh" "tarbitrator" "set_core.sh") + for i in "${bin_dir[@]}";do + check_file ${sbin_dir} $i + done + lbin_dir=("taos" "taosd" "taosadapter" "taosdemo" "rmtaos" "tarbitrator" "set_core") + for i in "${lbin_dir[@]}";do + check_link ${bin_link_dir}/$i + done + if [ "$verMode" == "cluster" ]; then + check_file ${nginx_dir}/sbin nginx + fi + echo -e "Check bin path:\033[32mOK\033[0m!" +} + +function check_lib_path() { + # check all links + check_link ${lib_link_dir}/libtaos.so + check_link ${lib_link_dir}/libtaos.so.1 + + if [[ -d ${lib64_link_dir} ]]; then + check_link ${lib64_link_dir}/libtaos.so + check_link ${lib64_link_dir}/libtaos.so.1 + fi + echo -e "Check lib path:\033[32mOK\033[0m!" +} + +function check_header_path() { + # check all header + header_dir=("taos.h" "taosdef.h" "taoserror.h") + for i in "${header_dir[@]}";do + check_link ${inc_link_dir}/$i + done + echo -e "Check bin path:\033[32mOK\033[0m!" +} + +function check_taosadapter_config_dir() { + # check all config + check_file ${cfg_install_dir} taosadapter.toml + check_file ${cfg_install_dir} taosadapter.service + check_file ${install_main_dir}/cfg taosadapter.toml.org + echo -e "Check conf path:\033[32mOK\033[0m!" +} + +function check_config_dir() { + # check all config + check_file ${cfg_install_dir} taos.cfg + check_file ${install_main_dir}/cfg taos.cfg.org + echo -e "Check conf path:\033[32mOK\033[0m!" +} + +function check_log_path() { + # check log path + check_file ${log_dir} + echo -e "Check log path:\033[32mOK\033[0m!" +} + +function check_data_path() { + # check data path + check_file ${data_dir} + echo -e "Check data path:\033[32mOK\033[0m!" +} + +function install_TDengine() { + cd ${script_dir} + tar zxf $1 + temp_version=$(get_package_name $1) + cd $(get_package_name $1) + echo -e "\033[32muninstall TDengine && install TDengine...\033[0m" + rmtaos >/dev/null 2>&1 || echo 'taosd not installed' && echo -e '\n\n' |./install.sh >/dev/null 2>&1 + echo -e "\033[32mTDengine has been installed!\033[0m" + echo -e "\033[32mTDengine is starting...\033[0m" + kill_process taos && systemctl start taosd && sleep 10 +} + +function test_TDengine() { + check_main_path + check_bin_path + check_lib_path + check_header_path + check_config_dir + check_taosadapter_config_dir + check_log_path + check_data_path + result=`taos -s 'create database test ;create table test.tt(ts timestamp ,i int);insert into test.tt values(now,11);select * from test.tt' 2>&1 ||:` + if [[ $result =~ "Unable to establish" ]];then + echo -e "\033[31mTDengine connect failed\033[0m" + fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n" + echo -e $fin_result + exit 8 + fi + echo -e "Check TDengine connect:\033[32mOK\033[0m!" + fin_result=$fin_result"\033[32m$temp_version\033[0m test OK!\n" +} +# ## ==============================Main program starts from here============================ +TD_package_name=`ls ${script_dir}/*server*gz |awk -F '/' '{print $NF}' ` +temp=`pwd` +for i in $TD_package_name;do + if [[ $i =~ 'enterprise' ]];then + verMode="cluster" + else + verMode="" + fi + cd $temp + install_TDengine $i + test_TDengine +done +echo "============================================================" +echo -e $fin_result diff --git a/packaging/deb/DEBIAN/control b/packaging/deb/DEBIAN/control new file mode 100644 index 0000000000000000000000000000000000000000..c3136e8f0d443f05be2905702883e065dd1b9eaa --- /dev/null +++ b/packaging/deb/DEBIAN/control @@ -0,0 +1,13 @@ +Package: tdengine +Version: 1.0.0 +Section: utils +Priority: optional +#Essential: no +#Depends: no +#Suggests: no +Architecture: amd64 +Installed-Size: 66666 +Maintainer: support@taosdata.com +Provides: taosdata +Homepage: http://taosdata.com +Description: Big Data Platform Designed and Optimized for IoT. diff --git a/packaging/deb/DEBIAN/postinst b/packaging/deb/DEBIAN/postinst new file mode 100644 index 0000000000000000000000000000000000000000..a6c2817082f484d1a533ff1d82a3500cba1b5c19 --- /dev/null +++ b/packaging/deb/DEBIAN/postinst @@ -0,0 +1,13 @@ +#!/bin/bash +#set -x +#path=`pwd` +insmetaPath="/usr/local/taos/script" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +${csudo}chmod -R 744 ${insmetaPath} +cd ${insmetaPath} +${csudo}./post.sh diff --git a/packaging/deb/DEBIAN/postrm b/packaging/deb/DEBIAN/postrm new file mode 100644 index 0000000000000000000000000000000000000000..98b0a844a8b7c1707e629a72962308e450ef510a --- /dev/null +++ b/packaging/deb/DEBIAN/postrm @@ -0,0 +1,2 @@ +#!/bin/bash + diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst new file mode 100644 index 0000000000000000000000000000000000000000..061f35538be4f48d0fcfc3d53ba7b61c2505a0fd --- /dev/null +++ b/packaging/deb/DEBIAN/preinst @@ -0,0 +1,40 @@ +#!/bin/bash + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +# Stop the service if running +if pidof taosd &> /dev/null; then + if pidof systemd &> /dev/null; then + ${csudo}systemctl stop taosd || : + elif $(which service &> /dev/null); then + ${csudo}service taosd stop || : + else + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi + echo "Stop taosd service success!" + sleep 1 +fi + +# if taos.cfg already softlink, remove it +cfg_install_dir="/etc/taos" +install_main_dir="/usr/local/taos" +if [ -f "${install_main_dir}/taos.cfg" ]; then + ${csudo}rm -f ${install_main_dir}/cfg/taos.cfg || : +fi + +if [ -f "${install_main_dir}/taosadapter.toml" ]; then + ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.toml || : +fi + +if [ -f "${install_main_dir}/taosadapter.service" ]; then + ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.service || : +fi + +# there can not libtaos.so*, otherwise ln -s error +${csudo}rm -f ${install_main_dir}/driver/libtaos* || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm new file mode 100644 index 0000000000000000000000000000000000000000..4298b33355ede507190ca891989ba6cd1fcefbb1 --- /dev/null +++ b/packaging/deb/DEBIAN/prerm @@ -0,0 +1,42 @@ +#!/bin/bash + +insmetaPath="/usr/local/taos/script" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +${csudo}chmod -R 744 ${insmetaPath} || : +#cd ${insmetaPath} +#${csudo}./preun.sh +if [ -f ${insmetaPath}/preun.sh ]; then + cd ${insmetaPath} + ${csudo}./preun.sh +else + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + inc_link_dir="/usr/include" + + data_link_dir="/usr/local/taos/data" + log_link_dir="/usr/local/taos/log" + cfg_link_dir="/usr/local/taos/cfg" + + # Remove all links + ${csudo}rm -f ${bin_link_dir}/taos || : + ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${bin_link_dir}/taosdemo || : + ${csudo}rm -f ${cfg_link_dir}/* || : + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + + ${csudo}rm -f ${log_link_dir} || : + ${csudo}rm -f ${data_link_dir} || : + + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +fi + diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh new file mode 100644 index 0000000000000000000000000000000000000000..e43c1a35698f24e776ae87d0e7ff0737889caeb4 --- /dev/null +++ b/packaging/deb/makedeb.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# Generate deb package for ubuntu +set -e +# set -x + +#curr_dir=$(pwd) +compile_dir=$1 +output_dir=$2 +tdengine_ver=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" +pkg_dir="${top_dir}/debworkroom" + +#echo "curr_dir: ${curr_dir}" +#echo "top_dir: ${top_dir}" +#echo "script_dir: ${script_dir}" +echo "compile_dir: ${compile_dir}" +echo "pkg_dir: ${pkg_dir}" + +if [ -d ${pkg_dir} ]; then + rm -rf ${pkg_dir} +fi +mkdir -p ${pkg_dir} +cd ${pkg_dir} + +libfile="libtaos.so.${tdengine_ver}" + +# create install dir +install_home_path="/usr/local/taos" +mkdir -p ${pkg_dir}${install_home_path} +mkdir -p ${pkg_dir}${install_home_path}/bin +mkdir -p ${pkg_dir}${install_home_path}/cfg +#mkdir -p ${pkg_dir}${install_home_path}/connector +mkdir -p ${pkg_dir}${install_home_path}/driver +mkdir -p ${pkg_dir}${install_home_path}/examples +mkdir -p ${pkg_dir}${install_home_path}/include +#mkdir -p ${pkg_dir}${install_home_path}/init.d +mkdir -p ${pkg_dir}${install_home_path}/script + +cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg +if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then + cp ${compile_dir}/test/cfg/taosadapter.toml ${pkg_dir}${install_home_path}/cfg || : +fi +if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then + cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || : +fi + +#cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d +cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script +cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script +cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin + +cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin +#cp ${compile_dir}/build/bin/taosBenchmark ${pkg_dir}${install_home_path}/bin + +if [ -f "${compile_dir}/build/bin/taosadapter" ]; then + cp ${compile_dir}/build/bin/taosadapter ${pkg_dir}${install_home_path}/bin ||: +fi + +cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver +cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include +cp ${compile_dir}/../src/inc/taosdef.h ${pkg_dir}${install_home_path}/include +cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include +cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples +#cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector +#cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector +#cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector +#cp ${compile_dir}/build/lib/taos-jdbcdriver*.* ${pkg_dir}${install_home_path}/connector ||: + +install_user_local_path="/usr/local" + +if [ -f ${compile_dir}/build/bin/jemalloc-config ]; then + mkdir -p ${pkg_dir}${install_user_local_path}/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${compile_dir}/build/bin/jemalloc-config ${pkg_dir}${install_user_local_path}/bin/ + if [ -f ${compile_dir}/build/bin/jemalloc.sh ]; then + cp ${compile_dir}/build/bin/jemalloc.sh ${pkg_dir}${install_user_local_path}/bin/ + fi + if [ -f ${compile_dir}/build/bin/jeprof ]; then + cp ${compile_dir}/build/bin/jeprof ${pkg_dir}${install_user_local_path}/bin/ + fi + if [ -f ${compile_dir}/build/include/jemalloc/jemalloc.h ]; then + cp ${compile_dir}/build/include/jemalloc/jemalloc.h ${pkg_dir}${install_user_local_path}/include/jemalloc/ + fi + if [ -f ${compile_dir}/build/lib/libjemalloc.so.2 ]; then + cp ${compile_dir}/build/lib/libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/ + ln -sf libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/libjemalloc.so + fi + if [ -f ${compile_dir}/build/lib/libjemalloc.a ]; then + cp ${compile_dir}/build/lib/libjemalloc.a ${pkg_dir}${install_user_local_path}/lib/ + fi + if [ -f ${compile_dir}/build/lib/libjemalloc_pic.a ]; then + cp ${compile_dir}/build/lib/libjemalloc_pic.a ${pkg_dir}${install_user_local_path}/lib/ + fi + if [ -f ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ]; then + cp ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ${pkg_dir}${install_user_local_path}/lib/pkgconfig/ + fi + if [ -f ${compile_dir}/build/share/doc/jemalloc/jemalloc.html ]; then + cp ${compile_dir}/build/share/doc/jemalloc/jemalloc.html ${pkg_dir}${install_user_local_path}/share/doc/jemalloc/ + fi + if [ -f ${compile_dir}/build/share/man/man3/jemalloc.3 ]; then + cp ${compile_dir}/build/share/man/man3/jemalloc.3 ${pkg_dir}${install_user_local_path}/share/man/man3/ + fi +fi + +cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/ +chmod 755 ${pkg_dir}/DEBIAN/* + +# modify version of control +debver="Version: "$tdengine_ver +sed -i "2c$debver" ${pkg_dir}/DEBIAN/control + +#get taos version, then set deb name +if [ "$verMode" == "cluster" ]; then + debname="TDengine-server-"${tdengine_ver}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + debname="TDengine-server"-${tdengine_ver}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + debname="TDengine-server-"${tdengine_ver}-${verType}-${osType}-${cpuType}".deb" +elif [ "$verType" == "stable" ]; then + debname=${debname}".deb" +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +# make deb package +dpkg -b ${pkg_dir} $debname +echo "make deb package success!" + +cp ${pkg_dir}/*.deb ${output_dir} + +# clean temp dir +rm -rf ${pkg_dir} diff --git a/packaging/deb/taosd b/packaging/deb/taosd new file mode 100644 index 0000000000000000000000000000000000000000..4dd176d0975f552ce44fd989d4c49bcee2ba895b --- /dev/null +++ b/packaging/deb/taosd @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Modified from original source: Elastic Search +# https://github.com/elasticsearch/elasticsearch +# Thank you to the Elastic Search authors +# +# chkconfig: 2345 99 01 +# +### BEGIN INIT INFO +# Provides: TDengine +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts TDengine taosd +# Description: Starts TDengine taosd, a time-series database engine +### END INIT INFO + +set -e + +PATH="/bin:/usr/bin:/sbin:/usr/sbin" +NAME="TDengine" +USER="root" +GROUP="root" +DAEMON="/usr/local/taos/bin/taosd" +DAEMON_OPTS="" + +HTTPD_NAME="taosadapter" +DAEMON_HTTPD_NAME=$HTTPD_NAME +DAEMON_HTTPD="/usr/local/taos/bin/$HTTPD_NAME" + +PID_FILE="/var/run/$NAME.pid" +APPARGS="" + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +. /lib/lsb/init-functions + +case "$1" in + start) + + log_action_begin_msg "Starting TDengine..." + $DAEMON_HTTPD & + if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then + + touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE" + + if [ -n "$MAX_OPEN_FILES" ]; then + ulimit -n $MAX_OPEN_FILES + fi + + start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS + + log_end_msg $? + fi + ;; + + stop) + log_action_begin_msg "Stopping TDengine..." + pkill -9 $DAEMON_HTTPD_NAME + set +e + if [ -f "$PID_FILE" ]; then + start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null + if [ $? -eq 1 ]; then + log_action_cont_msg "TSD is not running but pid file exists, cleaning up" + elif [ $? -eq 3 ]; then + PID="`cat $PID_FILE`" + log_failure_msg "Failed to stop TDengine (pid $PID)" + exit 1 + fi + rm -f "$PID_FILE" + else + log_action_cont_msg "TDengine was not running" + fi + log_action_end_msg 0 + set -e + ;; + + restart|force-reload) + if [ -f "$PID_FILE" ]; then + $0 stop + sleep 1 + fi + $0 start + ;; + status) + status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME" + ;; + *) + exit 1 + ;; +esac + +exit 0 diff --git a/packaging/deb/tarbitratord b/packaging/deb/tarbitratord new file mode 100644 index 0000000000000000000000000000000000000000..8ece6cef062d082614827ab50627327aafc79e86 --- /dev/null +++ b/packaging/deb/tarbitratord @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Modified from original source: Elastic Search +# https://github.com/elasticsearch/elasticsearch +# Thank you to the Elastic Search authors +# +# chkconfig: 2345 99 01 +# +### BEGIN INIT INFO +# Provides: taoscluster +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts taoscluster tarbitrator +# Description: Starts taoscluster tarbitrator, a arbitrator +### END INIT INFO + +set -e + +PATH="/bin:/usr/bin:/sbin:/usr/sbin" +NAME="taoscluster" +USER="root" +GROUP="root" +DAEMON="/usr/local/taos/bin/tarbitrator" +DAEMON_OPTS="" +PID_FILE="/var/run/$NAME.pid" +APPARGS="" + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +. /lib/lsb/init-functions + +case "$1" in + start) + + log_action_begin_msg "Starting tarbitrator..." + if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then + + touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE" + + if [ -n "$MAX_OPEN_FILES" ]; then + ulimit -n $MAX_OPEN_FILES + fi + + start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS + + log_end_msg $? + fi + ;; + + stop) + log_action_begin_msg "Stopping tarbitrator..." + set +e + if [ -f "$PID_FILE" ]; then + start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null + if [ $? -eq 1 ]; then + log_action_cont_msg "TSD is not running but pid file exists, cleaning up" + elif [ $? -eq 3 ]; then + PID="`cat $PID_FILE`" + log_failure_msg "Failed to stop tarbitrator (pid $PID)" + exit 1 + fi + rm -f "$PID_FILE" + else + log_action_cont_msg "tarbitrator was not running" + fi + log_action_end_msg 0 + set -e + ;; + + restart|force-reload) + if [ -f "$PID_FILE" ]; then + $0 stop + sleep 1 + fi + $0 start + ;; + status) + status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME" + ;; + *) + exit 1 + ;; +esac + +exit 0 diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..58ba6e3f42af9f11ccb0e71333979826c20cfe85 --- /dev/null +++ b/packaging/docker/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:18.04 + +WORKDIR /root + +ARG pkgFile +ARG dirName +ARG cpuType +RUN echo ${pkgFile} && echo ${dirName} + +COPY ${pkgFile} /root/ +RUN tar -zxf ${pkgFile} +WORKDIR /root/ +RUN cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root +RUN rm /root/${pkgFile} +RUN rm -rf /root/${dirName} + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get clean && apt-get update && apt-get install -y locales tzdata netcat && locale-gen en_US.UTF-8 +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \ + LC_CTYPE=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 + +COPY ./bin/* /usr/bin/ + +ENV TINI_VERSION v0.19.0 +RUN bash -c 'echo -e "Downloading tini-${cpuType} ..."' +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--", "/usr/bin/entrypoint.sh"] +CMD ["taosd"] +VOLUME [ "/var/lib/taos", "/var/log/taos", "/corefile" ] diff --git a/packaging/docker/README.md b/packaging/docker/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2722dbfef5c45343d5fdd8f777762e63ae31bec7 --- /dev/null +++ b/packaging/docker/README.md @@ -0,0 +1,664 @@ +# TDengine Docker Image Quick Reference + +## What is TDengine? + +TDengine is an open-sourced big data platform under [GNU AGPL v3.0](http://www.gnu.org/licenses/agpl-3.0.html), designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and cost of development and operation. + +- **10x Faster on Insert/Query Speeds**: Through the innovative design on storage, on a single-core machine, over 20K requests can be processed, millions of data points can be ingested, and over 10 million data points can be retrieved in a second. It is 10 times faster than other databases. + +- **1/5 Hardware/Cloud Service Costs**: Compared with typical big data solutions, less than 1/5 of computing resources are required. Via column-based storage and tuned compression algorithms for different data types, less than 1/10 of storage space is needed. + +- **Full Stack for Time-Series Data**: By integrating a database with message queuing, caching, and stream computing features together, it is no longer necessary to integrate Kafka/Redis/HBase/Spark or other software. It makes the system architecture much simpler and more robust. + +- **Powerful Data Analysis**: Whether it is 10 years or one minute ago, data can be queried just by specifying the time range. Data can be aggregated over time, multiple time streams or both. Ad Hoc queries or analyses can be executed via TDengine shell, Python, R or Matlab. + +- **Seamless Integration with Other Tools**: Telegraf, Grafana, Matlab, R, and other tools can be integrated with TDengine without a line of code. MQTT, OPC, Hadoop, Spark, and many others will be integrated soon. + +- **Zero Management, No Learning Curve**: It takes only seconds to download, install, and run it successfully; there are no other dependencies. Automatic partitioning on tables or DBs. Standard SQL is used, with C/C++, Python, JDBC, Go and RESTful connectors. + +## How to use this image + +### Start a TDengine instance with RESTful API exposed + +Simply, you can use `docker run` to start a TDengine instance and connect it with restful connectors(eg. [JDBC-RESTful](https://www.taosdata.com/cn/documentation/connector/java)). + +```bash +docker run -d --name tdengine -p 6041:6041 tdengine/tdengine +``` + +This command starts a docker container by name `tdengine` with TDengine server running, and maps the container's HTTP port 6041 to the host's port 6041. If you have `curl` in your host, you can list the databases by the command: + +```bash +curl -u root:taosdata -d "show databases" localhost:6041/rest/sql +``` + +You can execute the `taos` shell command in the container: + +```bash +$ docker exec -it tdengine taos + +Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show databases; + name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | +==================================================================================================================================================================================================================================================================================== + log | 2022-01-17 13:57:22.270 | 10 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | +Query OK, 1 row(s) in set (0.002843s) +``` + +Since TDengine use container hostname to establish connections, it's a bit more complex to use taos shell and native connectors(such as JDBC-JNI) with TDengine container instance. This is the recommended way to expose ports and use TDengine with docker in simple cases. If you want to use taos shell or taosc/connectors smoothly outside the `tdengine` container, see next use cases that match you need. + +### Start with host network + +```bash +docker run -d --name tdengine --network host tdengine/tdengine +``` + +Starts container with `host` network will use host's hostname as fqdn instead of container id. It's much like starting natively with `systemd` in host. After installing the client, you can use `taos` shell as normal in host path. + +```bash +$ taos + +Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | host:6030 | 1 | 8 | ready | any | 2022-01-17 22:10:32.619 | | +Query OK, 1 row(s) in set (0.003233s) +``` + +### Start with exposed ports and specified hostname + +Set the fqdn explicitly will help you to use in other environment or applications. We provide environment variable `TAOS_FQDN` or `fqdn` config option to explicitly set the hostname used by TDengine container instance(s). + +Use `TAOS_FQDN` variable within `docker run` command: + +```bash +docker run -d \ + --name tdengine \ + -e TAOS_FQDN=tdengine \ + -p 6030-6049:6030-6049 \ + -p 6030-6049:6030-6049/udp \ + tdengine/tdengine +``` + +This command starts a docker container with TDengine server running and maps the container's TCP ports from 6030 to 6049 to the host's ports from 6030 to 6049 with TCP protocol and UDP ports range 6030-6039 to the host's UDP ports 6030-6039. If the host is already running TDengine server and occupying the same port(s), you need to map the container's port to a different unused port segment. (Please see TDengine 2.0 Port Description for details). In order to support TDengine clients accessing TDengine server services, both TCP and UDP ports need to be exposed by default(unless `rpcForceTcp` is set to `1`). + +If you want to use taos shell or native connectors([JDBC-JNI](https://www.taosdata.com/cn/documentation/connector/java), or [driver-go](https://github.com/taosdata/driver-go)), you need to make sure the `TAOS_FQDN` is resolvable at `/etc/hosts` or with custom DNS service. + +If you set the `TAOS_FQDN` to host's hostname, it will works as using `hosts` network like previous use case. Otherwise, like in `-e TAOS_FQDN=tdengine`, you can add the hostname record `tdengine` into `/etc/hosts` (use `127.0.0.1` here in host path, if use TDengine client/application in other hosts, you should set the right ip to the host eg. `192.168.10.1`(check the real ip in host with `hostname -i` or `ip route list default`) to make the TDengine endpoint resolvable): + +```bash +echo 127.0.0.1 tdengine |sudo tee -a /etc/hosts +``` + +Then you can use `taos` with the host `tdengine`: + +```bash +taos -h tdengine +``` + +Or develop/test applications with native connectors. As in python: + +```python +import taos; +conn = taos.connect(host = "tdengine") +res = conn.query("show databases") +for row in res.fetch_all_into_dict(): + print(row) +``` + +See the results: + +```bash +Python 3.8.10 (default, Nov 26 2021, 20:14:08) +[GCC 9.3.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import taos; +>>> conn = taos.connect(host = "tdengine") +>>> res = conn.query("show databases") +>>> for row in res.fetch_all_into_dict(): +... print(row) +... +{'name': 'log', 'created_time': datetime.datetime(2022, 1, 17, 22, 56, 2, 490000), 'ntables': 11, 'vgroups': 1, 'replica': 1, 'quorum': 1, 'days': 10, 'keep': '30', 'cache(MB)': 1, 'blocks': 3, 'minrows': 100, 'maxrows': 4096, 'wallevel': 1, 'fsync': 3000, 'comp': 2, 'cachelast': 0, 'precision': 'us', 'update': 0, 'status': 'ready'} +``` + +### Start with specific network + +Alternatively, you can use TDengine natively by using specific network. + +First, create network for TDengine server and client/application. + +```bash +docker network create td-net +``` + +Start TDengine instance with service name as fqdn (explicitly set with `TAOS_FQDN`): + +```bash +docker run -d --name tdengine --network td-net \ + -e TAOS_FQDN=tdengine \ + tdengine/tdengine +``` + +Start TDengine client in another container with the specific network: + +```bash +docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine tdengine/tdengine taos +# or +docker run --rm -it --network td-net -e tdengine/tdengine taos -h tdengine +``` + +When you build your application with docker, you should add the TDengine client in the dockerfile, as based on `ubuntu:20.04` image, install the client like this: + +```dockerfile +FROM ubuntu:20.04 +RUN apt-get update && apt-get install -y wget +ENV TDENGINE_VERSION=2.4.0.0 +RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && cd TDengine-client-${TDENGINE_VERSION} \ + && ./install_client.sh \ + && cd ../ \ + && rm -rf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz TDengine-client-${TDENGINE_VERSION} +## add your application next, eg. go, build it in builder stage, copy the binary to the runtime +#COPY --from=builder /path/to/build/app /usr/bin/ +#CMD ["app"] +``` + +Here is an Go example app: + + + + +```go +/* + * In this test program, we'll create a database and insert 4 records then select out. + */ +package main + +import ( + "database/sql" + "flag" + "fmt" + "time" + + _ "github.com/taosdata/driver-go/v2/taosSql" +) + +type config struct { + hostName string + serverPort string + user string + password string +} + +var configPara config +var taosDriverName = "taosSql" +var url string + +func init() { + flag.StringVar(&configPara.hostName, "h", "", "The host to connect to TDengine server.") + flag.StringVar(&configPara.serverPort, "p", "", "The TCP/IP port number to use for the connection to TDengine server.") + flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.") + flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.") + flag.Parse() +} + +func printAllArgs() { + fmt.Printf("============= args parse result: =============\n") + fmt.Printf("hostName: %v\n", configPara.hostName) + fmt.Printf("serverPort: %v\n", configPara.serverPort) + fmt.Printf("usr: %v\n", configPara.user) + fmt.Printf("password: %v\n", configPara.password) + fmt.Printf("================================================\n") +} + +func main() { + printAllArgs() + + url = "root:taosdata@/tcp(" + configPara.hostName + ":" + configPara.serverPort + ")/" + + taos, err := sql.Open(taosDriverName, url) + checkErr(err, "open database error") + defer taos.Close() + + taos.Exec("create database if not exists test") + taos.Exec("use test") + taos.Exec("create table if not exists tb1 (ts timestamp, a int)") + _, err = taos.Exec("insert into tb1 values(now, 0)(now+1s,1)(now+2s,2)(now+3s,3)") + checkErr(err, "failed to insert") + rows, err := taos.Query("select * from tb1") + checkErr(err, "failed to select") + + defer rows.Close() + for rows.Next() { + var r struct { + ts time.Time + a int + } + err := rows.Scan(&r.ts, &r.a) + if err != nil { + fmt.Println("scan error:\n", err) + return + } + fmt.Println(r.ts, r.a) + } +} + +func checkErr(err error, prompt string) { + if err != nil { + fmt.Println("ERROR: %s\n", prompt) + panic(err) + } +} +``` + + + + +Full version of dockerfile could be: + +```dockerfile +FROM golang:1.17.6-buster as builder +ENV TDENGINE_VERSION=2.4.0.0 +RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && cd TDengine-client-${TDENGINE_VERSION} \ + && ./install_client.sh \ + && cd ../ \ + && rm -rf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz TDengine-client-${TDENGINE_VERSION} +WORKDIR /usr/src/app/ +ENV GOPROXY="https://goproxy.io,direct" +COPY ./main.go ./go.mod ./go.sum /usr/src/app/ +RUN go env && go mod tidy && go build + +FROM ubuntu:20.04 +RUN apt-get update && apt-get install -y wget +ENV TDENGINE_VERSION=2.4.0.0 +RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ + && cd TDengine-client-${TDENGINE_VERSION} \ + && ./install_client.sh \ + && cd ../ \ + && rm -rf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz TDengine-client-${TDENGINE_VERSION} + +## add your application next, eg. go, build it in builder stage, copy the binary to the runtime +COPY --from=builder /usr/src/app/app /usr/bin/ +CMD ["app"] +``` + +Suppose you have `main.go`, `go.mod` `go.sum`, `app.dockerfile`, build the app and run it with network `td-net`: + +```bash +$ docker build -t app -f app.dockerfile +$ docker run --rm --network td-net app -h tdengine -p 6030 +============= args parse result: ============= +hostName: tdengine +serverPort: 6030 +usr: root +password: taosdata +================================================ +2022-01-17 15:56:55.48 +0000 UTC 0 +2022-01-17 15:56:56.48 +0000 UTC 1 +2022-01-17 15:56:57.48 +0000 UTC 2 +2022-01-17 15:56:58.48 +0000 UTC 3 +2022-01-17 15:58:01.842 +0000 UTC 0 +2022-01-17 15:58:02.842 +0000 UTC 1 +2022-01-17 15:58:03.842 +0000 UTC 2 +2022-01-17 15:58:04.842 +0000 UTC 3 +2022-01-18 01:43:48.029 +0000 UTC 0 +2022-01-18 01:43:49.029 +0000 UTC 1 +2022-01-18 01:43:50.029 +0000 UTC 2 +2022-01-18 01:43:51.029 +0000 UTC 3 +``` + +Now you must be much familiar with developing and testing with TDengine, let's see some more complex cases. + +### Start with docker-compose with multiple nodes(instances) + +Start a 2-replicas-2-mnodes-2-dnodes-1-arbitrator TDengine cluster with `docker-compose` is quite simple. Save the file as `docker-compose.yml`: + +```yaml +version: "3" +services: + arbitrator: + image: tdengine/tdengine:$VERSION + command: tarbitrator + td-1: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: +``` + +You may notice that: + +- We use `VERSION` environment variable to set `tdengine` image tag version once. +- **`TAOS_FIRST_EP`** **MUST** be set to join the newly created instances into an existing TDengine cluster. If you want more instances, use `TAOS_SECOND_EP` in case of HA(High Availability) concerns. +- `TAOS_NUM_OF_MNODES` is for setting number of mnodes for the cluster. +- `TAOS_REPLICA` set the default database replicas, `2` means there're one master and one slave copy of data. The `replica` option should be `1 <= replica <= 3`, and not greater than dnodes number. +- `TAOS_ARBITRATOR` set the arbitrator entrypoint of the cluster for failover/election stuff. It's better to use arbitrator in a two nodes cluster. +- The way to start an arbitrator service is as easy as abc: just add command name `tarbitrator`(which is the binary name of arbitrator daemon) in docker-compose service option: `command: tarbitrator`, and everything is ok now. + +Now run `docker-compose up -d` with version specified: + +```bash +$ VERSION=2.4.0.0 docker-compose up -d +Creating network "test_default" with the default driver +Creating volume "test_taosdata-td1" with default driver +Creating volume "test_taoslog-td1" with default driver +Creating volume "test_taosdata-td2" with default driver +Creating volume "test_taoslog-td2" with default driver +Creating test_td-1_1 ... done +Creating test_arbitrator_1 ... done +Creating test_td-2_1 ... done +``` + +Check the status: + +```bash +$ docker-compose ps + Name Command State Ports +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_arbitrator_1 /usr/bin/entrypoint.sh tar ... Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp +test_td-1_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp +test_td-2_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp +``` + +Check dnodes with taos shell: + +```bash +$ docker-compose exec td-1 taos -s "show dnodes" + +Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | td-1:6030 | 1 | 8 | ready | any | 2022-01-18 02:47:42.871 | | + 2 | td-2:6030 | 0 | 8 | ready | any | 2022-01-18 02:47:43.518 | | + 0 | arbitrator:6042 | 0 | 0 | ready | arb | 2022-01-18 02:47:43.633 | - | +Query OK, 3 row(s) in set (0.000811s) +``` + +### Start a TDengine cluster with scaled taosadapter service + +In previous use case, you could see the way to start other services built with TDengine(`taosd` as the default command). There's another important service you should know: + +> **taosAdapter** is a TDengine’s companion tool and is a bridge/adapter between TDengine cluster and application. It provides an easy-to-use and efficient way to ingest data from data collections agents(like Telegraf, StatsD, CollectD) directly. It also provides InfluxDB/OpenTSDB compatible data ingestion interface to allow InfluxDB/OpenTSDB applications to immigrate to TDengine seamlessly. + +`taosadapter` is running inside `tdengine` image by default, you can disable it by `TAOS_DISABLE_ADAPTER=true`. Running `taosadapter` in a separate container is like how `arbitrator` does: + +```yaml +services: + # ... + adapter: + image: tdengine/tdengine:$VERSION + command: taosadapter +``` + +`taosadapter` could be scaled with docker-compose, so that you can manage the `taosadapter` nodes easily. Here is an example shows 4-`taosadapter` instances in a TDengine cluster(much like previous use cases): + +```yaml +version: "3" + +networks: + inter: + api: + +services: + arbitrator: + image: tdengine/tdengine:$VERSION + command: tarbitrator + networks: + - inter + td-1: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ + adapter: + image: tdengine/tdengine:$VERSION + command: taosadapter + networks: + - inter + environment: + TAOS_FIRST_EP: "td-1" + TAOS_SECOND_EP: "td-2" + deploy: + replicas: 4 + nginx: + image: nginx + depends_on: + - adapter + networks: + - inter + - api + ports: + - 6041:6041 + - 6044:6044/udp + command: [ + "sh", + "-c", + "while true; + do curl -s http://adapter:6041/-/ping >/dev/null && break; + done; + printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' + > /etc/nginx/conf.d/rest.conf; + printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' + >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; + nginx -g 'daemon off;'", + ] +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: +``` + +Start the cluster: + +```bash +$ VERSION=2.4.0.0 docker-compose up -d +Creating network "docker_inter" with the default driver +Creating network "docker_api" with the default driver +Creating volume "docker_taosdata-td1" with default driver +Creating volume "docker_taoslog-td1" with default driver +Creating volume "docker_taosdata-td2" with default driver +Creating volume "docker_taoslog-td2" with default driver +Creating docker_td-2_1 ... done +Creating docker_arbitrator_1 ... done +Creating docker_td-1_1 ... done +Creating docker_adapter_1 ... done +Creating docker_adapter_2 ... done +Creating docker_adapter_3 ... done +``` + +It will start a TDengine cluster with two dnodes and four taosadapter instances, expose ports 6041/tcp and 6044/udp to host. + +`6041` is the RESTful API endpoint port, you can verify that the RESTful interface taosAdapter provides working using the `curl` command. + +```bash +$ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' 127.0.0.1:6041/rest/sql +{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[["log","2022-01-18 04:37:42.902",16,1,1,1,10,"30",1,3,100,4096,1,3000,2,0,"us",0,"ready"]],"rows":1} +``` + +If you run curl in batch(here we use [hyperfine](https://github.com/sharkdp/hyperfine) - a command-line benchmarking tool), the requests are balanced into 4 adapter instances. + +```bash +hyperfine -m10 'curl -u root:taosdata localhost:6041/rest/sql -d "describe log.log"' +``` + +View the logs with `docker-compose logs`: + +```bash +$ docker-compose logs adapter +# some logs skipped +adapter_2 | 01/18 04:57:44.616529 00000039 TAOS_ADAPTER info "| 200 | 162.185µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 +adapter_1 | 01/18 04:57:44.627695 00000039 TAOS_ADAPTER info "| 200 | 145.485µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=17 +adapter_3 | 01/18 04:57:44.639165 00000040 TAOS_ADAPTER info "| 200 | 146.913µs | 172.21.0.9 | POST | /rest/sql " sessionID=17 model=web +adapter_4 | 01/18 04:57:44.650829 00000039 TAOS_ADAPTER info "| 200 | 153.201µs | 172.21.0.9 | POST | /rest/sql " sessionID=17 model=web +adapter_2 | 01/18 04:57:44.662422 00000039 TAOS_ADAPTER info "| 200 | 211.393µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 +adapter_1 | 01/18 04:57:44.673426 00000039 TAOS_ADAPTER info "| 200 | 154.714µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 +adapter_3 | 01/18 04:57:44.684788 00000040 TAOS_ADAPTER info "| 200 | 131.876µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 +adapter_4 | 01/18 04:57:44.696261 00000039 TAOS_ADAPTER info "| 200 | 162.173µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 +adapter_2 | 01/18 04:57:44.707414 00000039 TAOS_ADAPTER info "| 200 | 164.419µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 +adapter_1 | 01/18 04:57:44.720842 00000039 TAOS_ADAPTER info "| 200 | 179.374µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 +adapter_3 | 01/18 04:57:44.732184 00000040 TAOS_ADAPTER info "| 200 | 141.174µs | 172.21.0.9 | POST | /rest/sql " sessionID=19 model=web +adapter_4 | 01/18 04:57:44.744024 00000039 TAOS_ADAPTER info "| 200 | 159.774µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 +adapter_2 | 01/18 04:57:44.773732 00000039 TAOS_ADAPTER info "| 200 | 178.993µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=21 +adapter_1 | 01/18 04:57:44.796518 00000039 TAOS_ADAPTER info "| 200 | 238.24µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 +adapter_3 | 01/18 04:57:44.810744 00000040 TAOS_ADAPTER info "| 200 | 176.133µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 +adapter_4 | 01/18 04:57:44.826395 00000039 TAOS_ADAPTER info "| 200 | 149.215µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 +``` + +`6044/udp` is the [StatsD](https://github.com/statsd/statsd)-compatible port, you can verify this feature with `nc` command(usually provided by `netcat` package). + +```bash +echo "foo:1|c" | nc -u -w0 127.0.0.1 6044 +``` + +Check the result in `taos` shell with `docker-compose exec`: + +```bash +$ dc exec td-1 taos + +Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show databases; + name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | +==================================================================================================================================================================================================================================================================================== + log | 2022-01-18 04:37:42.902 | 17 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | + statsd | 2022-01-18 04:45:02.563 | 1 | 1 | 2 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | +Query OK, 2 row(s) in set (0.001838s) + +taos> select * from statsd.foo; + ts | value | metric_type | +======================================================================================= + 2022-01-18 04:45:02.563422822 | 1 | counter | +Query OK, 1 row(s) in set (0.003854s) +``` + +Use `docker-compose up -d adapter=1 to reduce the instances to 1 + +### Deploy TDengine cluster in Docker Swarm with `docker-compose.yml` + +If you use docker swarm mode, it will schedule arbitrator/taosd/taosadapter services into different hosts automatically. If you've no experience with k8s/kubernetes, this is the most convenient way to scale out the TDengine cluster with multiple hosts/servers. + +Use the `docker-compose.yml` file in previous use case, and deploy with `docker stack` or `docker deploy`: + +```bash +$ VERSION=2.4.0 docker stack deploy -c docker-compose.yml taos +Creating network taos_inter +Creating network taos_api +Creating service taos_arbitrator +Creating service taos_td-1 +Creating service taos_td-2 +Creating service taos_adapter +Creating service taos_nginx +``` + +Now you've created a TDengine cluster with multiple host servers. + +Use `docker service` or `docker stack` to manage the cluster: + + + +```bash +$ docker stack ps taos +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +79ni8temw59n taos_nginx.1 nginx:latest TM1701 Running Running about a minute ago +3e94u72msiyg taos_adapter.1 tdengine/tdengine:2.4.0 TM1702 Running Running 56 seconds ago +100amjkwzsc6 taos_td-2.1 tdengine/tdengine:2.4.0 TM1703 Running Running about a minute ago +pkjehr2vvaaa taos_td-1.1 tdengine/tdengine:2.4.0 TM1704 Running Running 2 minutes ago +tpzvgpsr1qkt taos_arbitrator.1 tdengine/tdengine:2.4.0 TM1705 Running Running 2 minutes ago +rvss3g5yg6fa taos_adapter.2 tdengine/tdengine:2.4.0 TM1706 Running Running 56 seconds ago +i2augxamfllf taos_adapter.3 tdengine/tdengine:2.4.0 TM1707 Running Running 56 seconds ago +lmjyhzccpvpg taos_adapter.4 tdengine/tdengine:2.4.0 TM1708 Running Running 56 seconds ago +$ docker service ls +ID NAME MODE REPLICAS IMAGE PORTS +561t4lu6nfw6 taos_adapter replicated 4/4 tdengine/tdengine:2.4.0 +3hk5ct3q90sm taos_arbitrator replicated 1/1 tdengine/tdengine:2.4.0 +d8qr52envqzu taos_nginx replicated 1/1 nginx:latest *:6041->6041/tcp, *:6044->6044/udp +2isssfvjk747 taos_td-1 replicated 1/1 tdengine/tdengine:2.4.0 +9pzw7u02ichv taos_td-2 replicated 1/1 tdengine/tdengine:2.4.0 +``` + + + +It shows that there are two dnodes, one arbitrator, four taosadapter and one nginx reverse-forward service in this cluster. + +You can scale down the taosadapter replicas to `1` by `docker service`: + +```bash +$ docker service scale taos_adapter=1 +taos_adapter scaled to 1 +overall progress: 1 out of 1 tasks +1/1: running [==================================================>] +verify: Service converged + +$ docker service ls -f name=taos_adapter +ID NAME MODE REPLICAS IMAGE PORTS +561t4lu6nfw6 taos_adapter replicated 1/1 tdengine/tdengine:2.4.0 +``` + +Now it remains only 1 taosadapter instance in the cluster. + +When you want to remove the cluster, just type: + +```bash +docker stack rm taos +``` + +### Environment Variables + +When you start `tdengine` image, you can adjust the configuration of TDengine by passing environment variables on the `docker run` command line or in the docker compose file. You can use all of the environment variables that passed to taosd or taosadapter. diff --git a/packaging/docker/bin/entrypoint.sh b/packaging/docker/bin/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..7173a35140d86b64161f57ec685f3fb1f7d4373d --- /dev/null +++ b/packaging/docker/bin/entrypoint.sh @@ -0,0 +1,83 @@ +#!/bin/sh +set -e +# for TZ awareness +if [ "$TZ" != "" ]; then + ln -sf /usr/share/zoneinfo/$TZ /etc/localtime + echo $TZ >/etc/timezone +fi + +# option to disable taosadapter, default is no +DISABLE_ADAPTER=${TAOS_DISABLE_ADAPTER:-0} +unset TAOS_DISABLE_ADAPTER + +# to get mnodeEpSet from data dir +DATA_DIR=${TAOS_DATA_DIR:-/var/lib/taos} + +# append env to custom taos.cfg +CFG_DIR=/tmp/taos +CFG_FILE=$CFG_DIR/taos.cfg + +mkdir -p $CFG_DIR >/dev/null 2>&1 + +[ -f /etc/taos/taos.cfg ] && cat /etc/taos/taos.cfg | grep -E -v "^#|^\s*$" >$CFG_FILE +env-to-cfg >>$CFG_FILE + +FQDN=$(cat $CFG_FILE | grep -E -v "^#|^$" | grep fqdn | tail -n1 | sed -E 's/.*fqdn\s+//') + +# ensure the fqdn is resolved as localhost +grep "$FQDN" /etc/hosts >/dev/null || echo "127.0.0.1 $FQDN" >>/etc/hosts + +# parse first ep host and port +FIRST_EP_HOST=${TAOS_FIRST_EP%:*} +FIRST_EP_PORT=${TAOS_FIRST_EP#*:} + +# in case of custom server port +SERVER_PORT=$(cat $CFG_FILE | grep -E -v "^#|^$" | grep serverPort | tail -n1 | sed -E 's/.*serverPort\s+//') +SERVER_PORT=${SERVER_PORT:-6030} + +# for other binaries like interpreters +if echo $1 | grep -E "taosd$" - >/dev/null; then + true # will run taosd +else + cp -f $CFG_FILE /etc/taos/taos.cfg || true + $@ + exit $? +fi + +set +e +ulimit -c unlimited +# set core files pattern, maybe failed +sysctl -w kernel.core_pattern=/corefile/core-$FQDN-%e-%p >/dev/null >&1 +set -e + +if [ "$DISABLE_ADAPTER" = "0" ]; then + which taosadapter >/dev/null && taosadapter & + # wait for 6041 port ready + for _ in $(seq 1 20); do + nc -z localhost 6041 && break + sleep 0.5 + done +fi + +# if has mnode ep set or the host is first ep or not for cluster, just start. +if [ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] || + [ "$TAOS_FQDN" = "$FIRST_EP_HOST" ]; then + $@ -c $CFG_DIR +# others will first wait the first ep ready. +else + if [ "$TAOS_FIRST_EP" = "" ]; then + echo "run TDengine with single node." + $@ -c $CFG_DIR + exit $? + fi + while true; do + es=0 + taos -h $FIRST_EP_HOST -P $FIRST_EP_PORT -n startup >/dev/null || es=$? + if [ "$es" -eq 0 ]; then + taos -h $FIRST_EP_HOST -P $FIRST_EP_PORT -s "create dnode \"$FQDN:$SERVER_PORT\";" + break + fi + sleep 1s + done + $@ -c $CFG_DIR +fi diff --git a/packaging/docker/bin/env-to-cfg b/packaging/docker/bin/env-to-cfg new file mode 100644 index 0000000000000000000000000000000000000000..b7741d02e1f0f902f4711571ce7eb7440f899525 --- /dev/null +++ b/packaging/docker/bin/env-to-cfg @@ -0,0 +1,13 @@ +#!/bin/sh +set -e +self=$0 + +snake_to_camel_case() { + echo $1 | awk -F _ '{printf "%s", $1; for(i=2; i<=NF; i++) printf "%s", toupper(substr($i,1,1)) substr($i,2); print"";}' +} + +if echo $1 | grep -E "^$" - >/dev/null; then + export |grep -E 'TAOS_.*' -o| sed 's/TAOS_//' |tr A-Z a-z | awk -F"=" '{print "name=$(""'$self' " $1"); echo $name "$2}' |sh +else + snake_to_camel_case $1 +fi diff --git a/packaging/docker/docker-compose.yml b/packaging/docker/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..e15ad0cf4f240ca6d51e621416c1d9ce945bc59d --- /dev/null +++ b/packaging/docker/docker-compose.yml @@ -0,0 +1,77 @@ +version: "3" + +networks: + inter: + api: + +services: + arbitrator: + image: tdengine/tdengine:$VERSION + command: tarbitrator + networks: + - inter + td-1: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + TAOS_NUM_OF_MNODES: "2" + TAOS_REPLICA: "2" + TAOS_ARBITRATOR: arbitrator:6042 + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ + adapter: + image: tdengine/tdengine:$VERSION + command: taosadapter + networks: + - inter + environment: + TAOS_FIRST_EP: "td-1" + TOAS_SECOND_EP: "td-2" + deploy: + replicas: 4 + update_config: + parallelism: 4 + nginx: + image: nginx + depends_on: + - adapter + networks: + - inter + - api + ports: + - 6041:6041 + - 6044:6044/udp + command: [ + "sh", + "-c", + "while true; + do curl -s http://adapter:6041/-/ping >/dev/null && break; + done; + printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' + > /etc/nginx/conf.d/rest.conf; + printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' + >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; + nginx -g 'daemon off;'", + ] +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh new file mode 100644 index 0000000000000000000000000000000000000000..b90e0e9a6561289bb64deb584a307d9c558c0886 --- /dev/null +++ b/packaging/docker/dockerManifest.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -e +#set -x + +# dockerbuild.sh +# -n [version number] +# -p [xxxx] +# -V [stable | beta] + +# set parameters by default value +version="" +passWord="" +verType="" + +while getopts "hn:p:V:" arg +do + case $arg in + n) + #echo "version=$OPTARG" + version=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + V) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "version=${version}" + +#docker manifest rm tdengine/tdengine +#docker manifest rm tdengine/tdengine:${version} +if [ "$verType" == "beta" ]; then + docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} + docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest + docker manifest rm tdengine/tdengine-beta:${version} + docker manifest rm tdengine/tdengine-beta:latest + docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} + docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest + docker manifest inspect tdengine/tdengine:latest + docker manifest inspect tdengine/tdengine:${version} + docker login -u tdengine -p ${passWord} #replace the docker registry username and password + docker manifest push tdengine/tdengine-beta:${version} + docker manifest push tdengine/tdengine-beta:latest +elif [ "$verType" == "stable" ]; then + docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} + docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + docker manifest rm tdengine/tdengine:latest + docker manifest rm tdengine/tdengine:${version} + docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} + docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + docker manifest inspect tdengine/tdengine:latest + docker manifest inspect tdengine/tdengine:${version} + docker login -u tdengine -p ${passWord} #replace the docker registry username and password + docker manifest push tdengine/tdengine:${version} + docker manifest push tdengine/tdengine:latest +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +# docker manifest create -a tdengine/${dockername}:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} +# docker manifest create -a tdengine/${dockername}:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + +# docker login -u tdengine -p ${passWord} #replace the docker registry username and password + +# docker manifest push tdengine/tdengine:latest + +# # how set latest version ??? diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh new file mode 100644 index 0000000000000000000000000000000000000000..c5e3ce15fb639bf58d96474e6a7f1f1454d27dbc --- /dev/null +++ b/packaging/docker/dockerbuild.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# + +set -e +#set -x + +# dockerbuild.sh +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -n [version number] +# -p [password for docker hub] +# -V [stable | beta] +# -f [pkg file] + +# set parameters by default value +cpuType="" +cpuTypeAlias="" +version="" +passWord="" +pkgFile="" +verType="stable" +dockerLatest="n" + +while getopts "hc:n:p:f:V:a:b:" arg +do + case $arg in + c) + #echo "cpuType=$OPTARG" + cpuType=$(echo $OPTARG) + ;; + n) + #echo "version=$OPTARG" + version=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + f) + #echo "pkgFile=$OPTARG" + pkgFile=$(echo $OPTARG) + ;; + b) + #echo "branchName=$OPTARG" + branchName=$(echo $OPTARG) + ;; + V) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + a) + #echo "dockerLatest=$OPTARG" + dockerLatest=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo " -n [version number] " + echo " -p [password for docker hub] " + echo " -V [stable | beta] " + echo " -f [pkg file] " + echo " -a [y | n ] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + + +# Check_verison() +# { +# } + + +if [ "$verType" == "beta" ]; then + dockername=${cpuType}-${verType} + dirName=${pkgFile%-beta*} +elif [ "$verType" == "stable" ]; then + dockername=${cpuType} + dirName=${pkgFile%-Linux*} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + + +echo "cpuType=${cpuType} version=${version} pkgFile=${pkgFile} verType=${verType} " +echo "$(pwd)" +echo "====NOTES: ${pkgFile} must be in the same directory as dockerbuild.sh====" + +scriptDir=$(dirname $(readlink -f $0)) +comunityArchiveDir=/nas/TDengine/v$version/community # community version’package directory +communityDir=${scriptDir}/../../../community +DockerfilePath=${communityDir}/packaging/docker/ +Dockerfile=${communityDir}/packaging/docker/Dockerfile +cd ${scriptDir} +cp -f ${comunityArchiveDir}/${pkgFile} . + +echo "dirName=${dirName}" + +if [[ "${cpuType}" == "x64" ]] || [[ "${cpuType}" == "amd64" ]]; then + cpuTypeAlias="amd64" +elif [[ "${cpuType}" == "aarch64" ]]; then + cpuTypeAlias="arm64" +elif [[ "${cpuType}" == "aarch32" ]]; then + cpuTypeAlias="armhf" +else + echo "Unknown cpuType: ${cpuType}" + exit 1 +fi + +docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias} +docker login -u tdengine -p ${passWord} #replace the docker registry username and password +docker push tdengine/tdengine-${dockername}:${version} + +if [ -n "$(docker ps -aq)" ] ;then + echo "delete docker process" + docker stop $(docker ps -aq) + docker rm $(docker ps -aq) +fi + +if [ -n "$(pidof taosd)" ] ;then + echo "kill taosd " + kill -9 $(pidof taosd) +fi + +if [ -n "$(pidof power)" ] ;then + echo "kill power " + kill -9 $(pidof power) +fi + + +echo ">>>>>>>>>>>>> check whether tdengine/tdengine-${dockername}:${version} has been published" +docker run -d --name doctest -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine-${dockername}:${version} +sleep 2 +curl -u root:taosdata -d 'show variables;' 127.0.0.1:6041/rest/sql > temp1.data +data_version=$( cat temp1.data |jq .data| jq '.[]' |grep "version" -A 2 -B 1 | jq ".[1]") +echo "${data_version}" +if [ "${data_version}" == "\"${version}\"" ] ; then + echo "docker version is right " +else + echo "docker version is wrong " + exit 1 +fi +rm -rf temp1.data + +# set this version to latest version +if [ ${dockerLatest} == 'y' ] ;then + docker tag tdengine/tdengine-${dockername}:${version} tdengine/tdengine-${dockername}:latest + docker push tdengine/tdengine-${dockername}:latest + echo ">>>>>>>>>>>>> check whether tdengine/tdengine-${dockername}:latest has been published correctly" + docker run -d --name doctestla -p 7030-7049:6030-6049 -p 7030-7049:6030-6049/udp tdengine/tdengine-${dockername}:latest + sleep 2 + curl -u root:taosdata -d 'show variables;' 127.0.0.1:7041/rest/sql > temp2.data + version_latest=` cat temp2.data |jq .data| jq '.[]' |grep "version" -A 2 -B 1 | jq ".[1]" ` + echo "${version_latest}" + if [ "${version_latest}" == "\"${version}\"" ] ; then + echo "docker version is right " + else + echo "docker version is wrong " + exit 1 + fi +fi +rm -rf temp2.data + +if [ -n "$(docker ps -aq)" ] ;then + echo "delte docker process" + docker stop $(docker ps -aq) + docker rm $(docker ps -aq) +fi + +cd ${scriptDir} +rm -f ${pkgFile} diff --git a/packaging/docker/dockerbuildi.sh b/packaging/docker/dockerbuildi.sh new file mode 100644 index 0000000000000000000000000000000000000000..318947929006958d52f9cd1585c85fd2c1baf84b --- /dev/null +++ b/packaging/docker/dockerbuildi.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# + +set -e +#set -x + +# dockerbuild.sh +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -n [version number] +# -p [password for docker hub] + +# set parameters by default value +cpuType=aarch64 +verNumber="" +passWord="" + +while getopts "hc:n:p:f:" arg +do + case $arg in + c) + #echo "cpuType=$OPTARG" + cpuType=$(echo $OPTARG) + ;; + n) + #echo "verNumber=$OPTARG" + verNumber=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo " -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +pkgFile=TDengine-server-${verNumber}-Linux-${cpuType}.tar.gz + +echo "cpuType=${cpuType} verNumber=${verNumber} pkgFile=${pkgFile} " + +scriptDir=`pwd` +pkgDir=$scriptDir/../../release/ + +cp -f ${pkgDir}/${pkgFile} . + +./dockerbuild.sh -c ${cpuType} -f ${pkgFile} -n ${verNumber} -p ${passWord} + +rm -f ${pkgFile} diff --git a/packaging/release.bat b/packaging/release.bat new file mode 100644 index 0000000000000000000000000000000000000000..a3fa575837d8bc95b2c4d6800c7bd119bdfc9c25 --- /dev/null +++ b/packaging/release.bat @@ -0,0 +1,62 @@ +@echo off + +set internal_dir=%~dp0\..\..\ +set community_dir=%~dp0\.. +cd %community_dir% +git checkout -- . +cd %community_dir%\packaging + +:: %1 name %2 version +if !%1==! GOTO USAGE +if !%2==! GOTO USAGE +if %1 == taos GOTO TAOS +if %1 == power GOTO POWER +if %1 == tq GOTO TQ +if %1 == pro GOTO PRO +if %1 == kh GOTO KH +if %1 == jh GOTO JH +GOTO USAGE + +:TAOS +goto RELEASE + +:POWER +call sed_power.bat %community_dir% +goto RELEASE + +:TQ +call sed_tq.bat %community_dir% +goto RELEASE + +:PRO +call sed_pro.bat %community_dir% +goto RELEASE + +:KH +call sed_kh.bat %community_dir% +goto RELEASE + +:JH +call sed_jh.bat %community_dir% +goto RELEASE + +:RELEASE +echo release windows-client-64 for %1, version: %2 +if not exist %internal_dir%\debug\ver-%2-64bit-%1 ( + md %internal_dir%\debug\ver-%2-64bit-%1 +) else ( + rd /S /Q %internal_dir%\debug\ver-%2-64bit-%1 + md %internal_dir%\debug\ver-%2-64bit-%1 +) +cd %internal_dir%\debug\ver-%2-64bit-%1 +call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 +cmake ../../ -G "NMake Makefiles" -DVERNUMBER=%2 -DCPUTYPE=x64 +set CL=/MP4 +nmake install +goto EXIT0 + +:USAGE +echo Usage: release.bat $productName $version +goto EXIT0 + +:EXIT0 \ No newline at end of file diff --git a/packaging/release.sh b/packaging/release.sh index a56d991bf889800ffbc5954e20fb1ebd28127b77..ef3018a913d54184ddb7703294e5e8759761ec96 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -21,17 +21,17 @@ echo "script_dir: ${script_dir}" echo "top_dir: ${top_dir}" cd ${top_dir} -git checkout -- . -git checkout 3.0 -git pull || : +# git checkout -- . +# git checkout 3.0 +# git pull || : echo "curr_dir: ${curr_dir}" # 2. cmake executable file compile_dir="${top_dir}/debug" -if [ -d ${compile_dir} ]; then - rm -rf ${compile_dir} -fi +# if [ -d ${compile_dir} ]; then +# rm -rf ${compile_dir} +# fi mkdir -p ${compile_dir} @@ -56,19 +56,19 @@ mkdir -p ${install_dir}/bin mkdir -p ${install_dir}/lib mkdir -p ${install_dir}/inc -install_files="${script_dir}/install.sh" -chmod a+x ${script_dir}/install.sh || : +install_files="${script_dir}/tools/install.sh" +chmod a+x ${script_dir}/tools/install.sh || : cp ${install_files} ${install_dir} header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h" cp ${header_files} ${install_dir}/inc -bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh ${compile_dir}/build/bin/taosBenchmark ${compile_dir}/build/bin/taosdump" +bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/tools/remove.sh ${compile_dir}/build/bin/taosBenchmark ${compile_dir}/build/bin/taosdump" cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : -cp -rf ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/ -cp -rf ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/ -cp -rf ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries" +cp ${compile_dir}/build/lib/libtaos.so ${install_dir}/lib/ +cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/ +cp ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries" cp -rf ${compile_dir}/build/lib/pkgconfig ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory" diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh new file mode 100644 index 0000000000000000000000000000000000000000..31673488e6fc8b4b226168f2ad95eb63af53fd72 --- /dev/null +++ b/packaging/rpm/makerpm.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Generate rpm package for centos + +set -e +# set -x + +#curr_dir=$(pwd) +compile_dir=$1 +output_dir=$2 +tdengine_ver=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" +pkg_dir="${top_dir}/rpmworkroom" +spec_file="${script_dir}/tdengine.spec" + +#echo "curr_dir: ${curr_dir}" +#echo "top_dir: ${top_dir}" +#echo "script_dir: ${script_dir}" +echo "compile_dir: ${compile_dir}" +echo "pkg_dir: ${pkg_dir}" +echo "spec_file: ${spec_file}" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +function cp_rpm_package() { + local cur_dir + cd $1 + cur_dir=$(pwd) + + for dirlist in "$(ls ${cur_dir})"; do + if test -d ${dirlist}; then + cd ${dirlist} + cp_rpm_package ${cur_dir}/${dirlist} + cd .. + fi + if test -e ${dirlist}; then + cp ${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm + fi + done +} + +if [ -d ${pkg_dir} ]; then + ${csudo}rm -rf ${pkg_dir} +fi +${csudo}mkdir -p ${pkg_dir} +cd ${pkg_dir} + +${csudo}mkdir -p BUILD BUILDROOT RPMS SOURCES SPECS SRPMS + +${csudo}rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_dir}" --define="_compiledir ${compile_dir}" -bb ${spec_file} + +# copy rpm package to output_dir, and modify package name, then clean temp dir +#${csudo}cp -rf RPMS/* ${output_dir} +cp_rpm_package ${pkg_dir}/RPMS + + +if [ "$verMode" == "cluster" ]; then + rpmname="TDengine-server-"${tdengine_ver}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + rpmname="TDengine-server"-${tdengine_ver}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + rpmname="TDengine-server-"${tdengine_ver}-${verType}-${osType}-${cpuType}".rpm" +elif [ "$verType" == "stable" ]; then + rpmname=${rpmname}".rpm" +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/${rpmname} + +cd .. +${csudo}rm -rf ${pkg_dir} diff --git a/packaging/rpm/taosd b/packaging/rpm/taosd new file mode 100644 index 0000000000000000000000000000000000000000..b79361f36e0f85085590acd81178bf7294222a54 --- /dev/null +++ b/packaging/rpm/taosd @@ -0,0 +1,145 @@ +#!/bin/bash +# +# taosd This shell script takes care of starting and stopping TDengine. +# +# chkconfig: 2345 99 01 +# description: TDengine is a districuted, scalable, high-performance Time Series Database +# (TSDB). More than just a pure database, TDengine also provides the ability +# to do stream computing, aggregation etc. +# +# +### BEGIN INIT INFO +# Provides: taosd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Short-Description: start and stop taosd +# Description: TDengine is a districuted, scalable, high-performance Time Series Database +# (TSDB). More than just a pure database, TDengine also provides the ability +# to do stream computing, aggregation etc. +### END INIT INFO + +# Source init functions +. /etc/init.d/functions + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +# Default program options +NAME=taosd +PROG=/usr/local/taos/bin/taosd +USER=root +GROUP=root + +# Default directories +LOCK_DIR=/var/lock/subsys +PID_DIR=/var/run/$NAME + +# Set file names +LOCK_FILE=$LOCK_DIR/$NAME +PID_FILE=$PID_DIR/$NAME.pid + +[ -e $PID_DIR ] || mkdir -p $PID_DIR + +PROG_OPTS="" + +start() { + echo -n "Starting ${NAME}: " + # check identity + curid="`id -u -n`" + if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then + echo "Must be run as root or $USER, but was run as $curid" + return 1 + fi + # Sets the maximum number of open file descriptors allowed. + ulimit -n $MAX_OPEN_FILES + curulimit="`ulimit -n`" + if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then + echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit" + return 1 + fi + + if [ "`id -u -n`" == root ] ; then + # Changes the owner of the lock, and the pid files to allow + # non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py. + touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE + touch $PID_FILE && chown $USER:$GROUP $PID_FILE + daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + else + # Don't have to change user. + daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + fi + retval=$? + sleep 2 + echo + [ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE) + return $retval +} + +stop() { + echo -n "Stopping ${NAME}: " + killproc -p $PID_FILE $NAME + retval=$? + echo + # Non-root users don't have enough permission to remove pid and lock files. + # So, the opentsdb_restart.py cannot get rid of the files, and the command + # "service opentsdb status" will complain about the existing pid file. + # Makes the pid file empty. + echo > $PID_FILE + [ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE) + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + # run checks to determine if the service is running or use generic status + status -p $PID_FILE -l $LOCK_FILE $NAME +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac + +exit $? diff --git a/packaging/rpm/tarbitratord b/packaging/rpm/tarbitratord new file mode 100644 index 0000000000000000000000000000000000000000..34fedab3d9330b0f6b88bf110e21a4d6f08955db --- /dev/null +++ b/packaging/rpm/tarbitratord @@ -0,0 +1,141 @@ +#!/bin/bash +# +# tarbitratord This shell script takes care of starting and stopping tarbitrator. +# +# chkconfig: 2345 99 01 +# description: tarbitrator is a arbitrator used in TDengine cluster. +# +# +### BEGIN INIT INFO +# Provides: taoscluster +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Short-Description: start and stop tarbitrator +# Description: tarbitrator is a arbitrator used in TDengine cluster. +### END INIT INFO + +# Source init functions +. /etc/init.d/functions + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +# Default program options +NAME=tarbitrator +PROG=/usr/local/taos/bin/tarbitrator +USER=root +GROUP=root + +# Default directories +LOCK_DIR=/var/lock/subsys +PID_DIR=/var/run/$NAME + +# Set file names +LOCK_FILE=$LOCK_DIR/$NAME +PID_FILE=$PID_DIR/$NAME.pid + +[ -e $PID_DIR ] || mkdir -p $PID_DIR + +PROG_OPTS="" + +start() { + echo -n "Starting ${NAME}: " + # check identity + curid="`id -u -n`" + if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then + echo "Must be run as root or $USER, but was run as $curid" + return 1 + fi + # Sets the maximum number of open file descriptors allowed. + ulimit -n $MAX_OPEN_FILES + curulimit="`ulimit -n`" + if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then + echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit" + return 1 + fi + + if [ "`id -u -n`" == root ] ; then + # Changes the owner of the lock, and the pid files to allow + # non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py. + touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE + touch $PID_FILE && chown $USER:$GROUP $PID_FILE + daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + else + # Don't have to change user. + daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + fi + retval=$? + sleep 2 + echo + [ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE) + return $retval +} + +stop() { + echo -n "Stopping ${NAME}: " + killproc -p $PID_FILE $NAME + retval=$? + echo + # Non-root users don't have enough permission to remove pid and lock files. + # So, the opentsdb_restart.py cannot get rid of the files, and the command + # "service opentsdb status" will complain about the existing pid file. + # Makes the pid file empty. + echo > $PID_FILE + [ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE) + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + # run checks to determine if the service is running or use generic status + status -p $PID_FILE -l $LOCK_FILE $NAME +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac + +exit $? diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec new file mode 100644 index 0000000000000000000000000000000000000000..022482c86749739427d532d13db8260ef78995fd --- /dev/null +++ b/packaging/rpm/tdengine.spec @@ -0,0 +1,236 @@ +%define homepath /usr/local/taos +%define userlocalpath /usr/local +%define cfg_install_dir /etc/taos +%define __strip /bin/true + +Name: tdengine +Version: %{_version} +Release: 3%{?dist} +Summary: tdengine from taosdata +Group: Application/Database +License: AGPL +URL: www.taosdata.com +AutoReqProv: no + +#BuildRoot: %_topdir/BUILDROOT +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root + +#Prefix: /usr/local/taos + +#BuildRequires: +#Requires: + +%description +Big Data Platform Designed and Optimized for IoT + +#"prep" Nothing needs to be done +#%prep +#%setup -q +#%setup -T + +#"build" Nothing needs to be done +#%build +#%configure +#make %{?_smp_mflags} + +%install +#make install DESTDIR=%{buildroot} +rm -rf %{buildroot} + +echo topdir: %{_topdir} +echo version: %{_version} +echo buildroot: %{buildroot} + +libfile="libtaos.so.%{_version}" + +# create install path, and cp file +mkdir -p %{buildroot}%{homepath}/bin +mkdir -p %{buildroot}%{homepath}/cfg +#mkdir -p %{buildroot}%{homepath}/connector +mkdir -p %{buildroot}%{homepath}/driver +mkdir -p %{buildroot}%{homepath}/examples +mkdir -p %{buildroot}%{homepath}/include +#mkdir -p %{buildroot}%{homepath}/init.d +mkdir -p %{buildroot}%{homepath}/script + +cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg +if [ -f %{_compiledir}/test/cfg/taosadapter.toml ]; then + cp %{_compiledir}/test/cfg/taosadapter.toml %{buildroot}%{homepath}/cfg +fi +if [ -f %{_compiledir}/test/cfg/taosadapter.service ]; then + cp %{_compiledir}/test/cfg/taosadapter.service %{buildroot}%{homepath}/cfg +fi +#cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d +cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script +cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/script +cp %{_compiledir}/../packaging/tools/startPre.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../packaging/tools/set_core.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../packaging/tools/taosd-dump-cfg.gdb %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin +#cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin + +if [ -f %{_compiledir}/build/bin/taosadapter ]; then + cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin ||: +fi +cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver +cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include +cp %{_compiledir}/../src/inc/taosdef.h %{buildroot}%{homepath}/include +cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include +#cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector +#cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector +#cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector +#cp %{_compiledir}/build/lib/taos-jdbcdriver*.* %{buildroot}%{homepath}/connector ||: +cp -r %{_compiledir}/../examples/* %{buildroot}%{homepath}/examples + +if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then + mkdir -p %{buildroot}%{userlocalpath}/bin + mkdir -p %{buildroot}%{userlocalpath}/lib + mkdir -p %{buildroot}%{userlocalpath}/lib/pkgconfig + mkdir -p %{buildroot}%{userlocalpath}/include + mkdir -p %{buildroot}%{userlocalpath}/include/jemalloc + mkdir -p %{buildroot}%{userlocalpath}/share + mkdir -p %{buildroot}%{userlocalpath}/share/doc + mkdir -p %{buildroot}%{userlocalpath}/share/doc/jemalloc + mkdir -p %{buildroot}%{userlocalpath}/share/man + mkdir -p %{buildroot}%{userlocalpath}/share/man/man3 + + cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{userlocalpath}/bin/ + if [ -f %{_compiledir}/build/bin/jemalloc.sh ]; then + cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{userlocalpath}/bin/ + fi + if [ -f %{_compiledir}/build/bin/jeprof ]; then + cp %{_compiledir}/build/bin/jeprof %{buildroot}%{userlocalpath}/bin/ + fi + if [ -f %{_compiledir}/build/include/jemalloc/jemalloc.h ]; then + cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{userlocalpath}/include/jemalloc/ + fi + if [ -f %{_compiledir}/build/lib/libjemalloc.so.2 ]; then + cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/ + ln -sf libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/libjemalloc.so + fi + if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then + cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{userlocalpath}/lib/ + fi + if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then + cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{userlocalpath}/lib/ + fi + if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then + cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{userlocalpath}/lib/pkgconfig/ + fi + if [ -f %{_compiledir}/build/share/doc/jemalloc/jemalloc.html ]; then + cp %{_compiledir}/build/share/doc/jemalloc/jemalloc.html %{buildroot}%{userlocalpath}/share/doc/jemalloc/ + fi + if [ -f %{_compiledir}/build/share/man/man3/jemalloc.3 ]; then + cp %{_compiledir}/build/share/man/man3/jemalloc.3 %{buildroot}%{userlocalpath}/share/man/man3/ + fi +fi + +#Scripts executed before installation +%pre +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +# Stop the service if running +if pidof taosd &> /dev/null; then + if pidof systemd &> /dev/null; then + ${csudo}systemctl stop taosd || : + elif $(which service &> /dev/null); then + ${csudo}service taosd stop || : + else + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi + echo "Stop taosd service success!" + sleep 1 +fi +# if taos.cfg already exist, remove it +if [ -f %{cfg_install_dir}/taos.cfg ]; then + ${csudo}rm -f %{cfg_install_dir}/cfg/taos.cfg || : +fi + +# if taosadapter.toml already exist, remove it +if [ -f %{cfg_install_dir}/taosadapter.toml ]; then + ${csudo}rm -f %{cfg_install_dir}/cfg/taosadapter.toml || : +fi + +# there can not libtaos.so*, otherwise ln -s error +${csudo}rm -f %{homepath}/driver/libtaos* || : + +#Scripts executed after installation +%post +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi +cd %{homepath}/script +${csudo}./post.sh + +# Scripts executed before uninstall +%preun +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi +# only remove package to call preun.sh, not but update(2) +if [ $1 -eq 0 ];then + #cd %{homepath}/script + #${csudo}./preun.sh + + if [ -f %{homepath}/script/preun.sh ]; then + cd %{homepath}/script + ${csudo}./preun.sh + else + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + inc_link_dir="/usr/include" + + data_link_dir="/usr/local/taos/data" + log_link_dir="/usr/local/taos/log" + cfg_link_dir="/usr/local/taos/cfg" + + # Remove all links + ${csudo}rm -f ${bin_link_dir}/taos || : + ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${cfg_link_dir}/* || : + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosdef.h || : + ${csudo}rm -f ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + + ${csudo}rm -f ${log_link_dir} || : + ${csudo}rm -f ${data_link_dir} || : + + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi +fi + +# Scripts executed after uninstall +%postun + +# clean build dir +%clean +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi +${csudo}rm -rf %{buildroot} + +#Specify the files to be packaged +%files +/* +#%doc + +#Setting default permissions +%defattr (-,root,root,0755) +#%{prefix} + +#%changelog diff --git a/packaging/tools/check_os.sh b/packaging/tools/check_os.sh new file mode 100644 index 0000000000000000000000000000000000000000..d5ec3326b03d93924c38c0e1373742f2fdf817d9 --- /dev/null +++ b/packaging/tools/check_os.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +CSI=$(echo -e "\033[") +CRED="${CSI}1;31m" +CFAILURE="$CRED" +CEND="${CSI}0m" +if [ -n "$(grep 'Aliyun Linux release' /etc/issue)" -o -e /etc/redhat-release ]; then + OS=CentOS + [ -n "$(grep ' 7\.' /etc/redhat-release 2> /dev/null)" ] && CentOS_RHEL_version=7 + [ -n "$(grep ' 6\.' /etc/redhat-release 2> /dev/null)" -o -n "$(grep 'Aliyun Linux release6 15' /etc/issue)" ] && CentOS_RHEL_version=6 + [ -n "$(grep ' 5\.' /etc/redhat-release 2> /dev/null)" -o -n "$(grep 'Aliyun Linux release5' /etc/issue)" ] && CentOS_RHEL_version=5 +elif [ -n "$(grep 'Amazon Linux AMI release' /etc/issue)" -o -e /etc/system-release ]; then + OS=CentOS + CentOS_RHEL_version=6 +elif [ -n "$(grep 'bian' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == "Debian" ]; then + OS=Debian + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + Debian_version=$(lsb_release -sr | awk -F. '{print $1}') +elif [ -n "$(grep 'Deepin' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == "Deepin" ]; then + OS=Debian + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + Debian_version=$(lsb_release -sr | awk -F. '{print $1}') +elif [ -n "$(grep 'Kali GNU/Linux Rolling' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == "Kali" ]; then + OS=Debian + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + if [ -n "$(grep 'VERSION="2016.*"' /etc/os-release)" ]; then + Debian_version=8 + else + echo "${CFAILURE}Does not support this OS, Please contact the author! ${CEND}" + kill -9 $$ + fi +elif [ -n "$(grep 'Ubuntu' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == "Ubuntu" -o -n "$(grep 'Linux Mint' /etc/issue)" ]; then + OS=Ubuntu + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + Ubuntu_version=$(lsb_release -sr | awk -F. '{print $1}') + [ -n "$(grep 'Linux Mint 18' /etc/issue)" ] && Ubuntu_version=16 +elif [ -n "$(grep 'elementary' /etc/issue)" -o "$(lsb_release -is 2>/dev/null)" == 'elementary' ]; then + OS=Ubuntu + [ ! -e "$(which lsb_release)" ] && { apt-get -y update; apt-get -y install lsb-release; clear; } + Ubuntu_version=16 +else + echo "${CFAILURE}Does not support this OS, Please contact the author! ${CEND}" + kill -9 $$ +fi + +echo "${CFAILURE}${OS}${CEND}" +if [ "$OS" == 'CentOS' ]; then + echo ${CentOS_RHEL_version} +else + echo ${Ubuntu_version} +fi + diff --git a/packaging/tools/get_client.sh b/packaging/tools/get_client.sh new file mode 100644 index 0000000000000000000000000000000000000000..c29e3e79e83520f370d99ef94e0be472eee23782 --- /dev/null +++ b/packaging/tools/get_client.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# + +log_dir=$1 +result_file=$2 + +if [ ! -n "$1" ];then + echo "Pleas input the director of taosdlog." + echo "usage: ./get_client.sh " + exit 1 +else + log_dir=$1 +fi + +if [ ! -n "$2" ];then + result_file=clientInfo.txt +else + result_file=$2 +fi + +grep "new TCP connection" ${log_dir}/taosdlog.* | sed -e "s/0x.* from / /"|sed -e "s/,.*$//"|sed -e "s/:[0-9]*$//"|sort -r|uniq -f 2|sort -k 3 -r|uniq -f 2 > ${result_file} diff --git a/packaging/tools/get_os.sh b/packaging/tools/get_os.sh new file mode 100644 index 0000000000000000000000000000000000000000..1216f7a6a6fa418357c59d00a1e8d09b83a19d27 --- /dev/null +++ b/packaging/tools/get_os.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# This file is used to install TAOS time-series database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +# set -x + +# -----------------------Variables definition--------------------- +OS=$(cat /etc/*-release | grep "^NAME=" | cut -d= -f2) +len=$(echo ${#OS}) +len=$((len-2)) +retval=$(echo -ne ${OS:1:${len}} | cut -d" " -f1) +echo -ne $retval diff --git a/packaging/tools/get_version.sh b/packaging/tools/get_version.sh new file mode 100644 index 0000000000000000000000000000000000000000..44c8c6bf21d93bbb9eba1157bd0fecf3e402b40c --- /dev/null +++ b/packaging/tools/get_version.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# This file is used to install TAOS time-series database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +# set -x + +# -----------------------Variables definition--------------------- +verinfo=$(cat $1 | grep " version" | cut -d '"' -f2) +verinfo=$(echo $verinfo | tr "\n" " ") +len=$(echo ${#verinfo}) +len=$((len-1)) +retval=$(echo -ne ${verinfo:0:${len}}) +echo -ne $retval diff --git a/packaging/install.sh b/packaging/tools/install.sh similarity index 100% rename from packaging/install.sh rename to packaging/tools/install.sh diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh new file mode 100644 index 0000000000000000000000000000000000000000..22874f058a919a0452fc303c9e69b3355444f604 --- /dev/null +++ b/packaging/tools/install_arbi.sh @@ -0,0 +1,339 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tarbitrator" + +# old bin dir +bin_dir="/usr/local/tarbitrator/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo >/dev/null; then + csudo="sudo " +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &>/dev/null; then + service_mod=0 +elif $(which service &>/dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &>/dev/null); then + initd_mod=1 + elif $(which insserv &>/dev/null); then + initd_mod=2 + elif $(which update-rc.d &>/dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) || : +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu"; then + # echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian"; then + # echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin"; then + # echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos"; then + # echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora"; then + # echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo}rm -rf ${install_main_dir} || : + ${csudo}mkdir -p ${install_main_dir} + ${csudo}mkdir -p ${install_main_dir}/bin + #${csudo}mkdir -p ${install_main_dir}/include + ${csudo}mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo}rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo}rm -f ${bin_link_dir}/tarbitrator || : + ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_arbi.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &>/dev/null; then + ${csudo}service tarbitratord stop || : + fi + + if ((${initd_mod} == 1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}chkconfig --del tarbitratord || : + fi + elif ((${initd_mod} == 2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}insserv -r tarbitratord || : + fi + elif ((${initd_mod} == 3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo}rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &>/dev/null); then + ${csudo}init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + if ((${os_type} == 1)); then + ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type} == 2)); then + ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord + fi + + if ((${initd_mod} == 1)); then + ${csudo}chkconfig --add tarbitratord || : + ${csudo}chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod} == 2)); then + ${csudo}insserv tarbitratord || : + ${csudo}insserv -d tarbitratord || : + elif ((${initd_mod} == 3)); then + ${csudo}update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo}systemctl stop tarbitratord &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null + + ${csudo}rm -f ${tarbitratord_service_config} +} + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo}bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo}systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod} == 0)); then + install_service_on_systemd + elif ((${service_mod} == 1)); then + install_service_on_sysvinit + else + kill_tarbitrator + fi +} + +function update_TDengine() { + # Start to update + echo -e "${GREEN}Start to update TDengine's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &>/dev/null; then + if ((${service_mod} == 0)); then + ${csudo}systemctl stop tarbitratord || : + elif ((${service_mod} == 1)); then + ${csudo}service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + install_jemalloc + + echo + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}systemctl start tarbitratord${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" +} + +function install_TDengine() { + # Start to install + echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + install_jemalloc + + echo + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}systemctl start tarbitratord${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mTDengine's arbitrator is installed successfully!${NC}" + echo +} + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update_TDengine +else + install_TDengine +fi diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh new file mode 100644 index 0000000000000000000000000000000000000000..5508f2b5d0d07ae1e967cae162bf351a43c5578f --- /dev/null +++ b/packaging/tools/install_client.sh @@ -0,0 +1,320 @@ +#!/bin/bash +# +# This file is used to install TDengine client on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- + +dataDir="/var/lib/taos" +logDir="/var/log/taos" +productName="TDengine" +installDir="/usr/local/taos" +configDir="/etc/taos" +serverName="taosd" +clientName="taos" +uninstallScript="rmtaos" +configFile="taos.cfg" +tarName="taos.tar.gz" + +osType=Linux +pagMode=full +verMode=edge + +if [ "$osType" != "Darwin" ]; then + script_dir=$(dirname $(readlink -f "$0")) + # Dynamic directory + data_dir=${dataDir} + log_dir=${logDir} +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + data_dir=${dataDir} + log_dir=~/${productName}/log +fi + +log_link_dir="${installDir}/log" + +cfg_install_dir=${configDir} + +if [ "$osType" != "Darwin" ]; then + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + lib64_link_dir="/usr/lib64" + inc_link_dir="/usr/include" +else + bin_link_dir="/usr/local/bin" + lib_link_dir="/usr/local/lib" + inc_link_dir="/usr/local/include" +fi + +#install main path +install_main_dir="${installDir}" + +# old bin dir +bin_dir="${installDir}/bin" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +update_flag=0 + +function kill_client() { + pid=$(ps -ef | grep "${clientName}" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo}rm -rf ${install_main_dir} || : + ${csudo}mkdir -p ${install_main_dir} + ${csudo}mkdir -p ${install_main_dir}/cfg + ${csudo}mkdir -p ${install_main_dir}/bin + ${csudo}mkdir -p ${install_main_dir}/driver + if [ $productName == "TDengine" ]; then + ${csudo}mkdir -p ${install_main_dir}/examples + fi + ${csudo}mkdir -p ${install_main_dir}/include + if [ "$verMode" == "cluster" ]; then + ${csudo}mkdir -p ${install_main_dir}/connector + fi +} + +function install_bin() { + # Remove links + ${csudo}rm -f ${bin_link_dir}/${clientName} || : + if [ "$osType" != "Darwin" ]; then + ${csudo}rm -f ${bin_link_dir}/taosdemo || : + fi + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/set_core || : + + ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : + if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + fi + [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : +} + +function clean_lib() { + sudo rm -f /usr/lib/libtaos.* || : + sudo rm -rf ${lib_dir} || : +} + +function install_lib() { + # Remove links + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo}rm -rf ${v15_java_app_dir} || : + + ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* + + if [ "$osType" != "Darwin" ]; then + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [ -d "${lib64_link_dir}" ]; then + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + else + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo}ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + fi + + if [ "$osType" != "Darwin" ]; then + ${csudo}ldconfig + else + ${csudo}update_dyld_shared_cache + fi +} + +function install_header() { + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf > /dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + +function install_config() { + if [ ! -f ${cfg_install_dir}/${configFile} ]; then + ${csudo}mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} + ${csudo}chmod 644 ${cfg_install_dir}/* + fi + + ${csudo}cp -f ${script_dir}/cfg/${configFile} ${install_main_dir}/cfg/${configFile}.org + ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg +} + + +function install_log() { + ${csudo}rm -rf ${log_dir} || : + + if [ "$osType" != "Darwin" ]; then + ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} + else + mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} + fi + ${csudo}ln -s ${log_dir} ${install_main_dir}/log +} + +function install_connector() { + ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function update_TDengine() { + # Start to update + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + tar -zxf ${tarName} + + echo -e "${GREEN}Start to update ${productName} client...${NC}" + # Stop the client shell if running + if pidof ${clientName} &> /dev/null; then + kill_client + sleep 1 + fi + + install_main_path + + install_log + install_header + install_lib + install_jemalloc + if [ "$verMode" == "cluster" ]; then + install_connector + fi + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}" + + rm -rf $(tar -tf ${tarName}) +} + +function install_TDengine() { + # Start to install + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + tar -zxf ${tarName} + + echo -e "${GREEN}Start to install ${productName} client...${NC}" + + install_main_path + install_log + install_header + install_lib + install_jemalloc + if [ "$verMode" == "cluster" ]; then + install_connector + fi + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}" + + rm -rf $(tar -tf ${tarName}) +} + + +## ==============================Main program starts from here============================ +# Install or updata client and client +# if server is already install, don't install client + if [ -e ${bin_dir}/${serverName} ]; then + echo -e "\033[44;32;1mThere are already installed ${productName} server, so don't need install client!${NC}" + exit 0 + fi + + if [ -x ${bin_dir}/${clientName} ]; then + update_flag=1 + update_TDengine + else + install_TDengine + fi diff --git a/packaging/make_install.sh b/packaging/tools/make_install.sh similarity index 97% rename from packaging/make_install.sh rename to packaging/tools/make_install.sh index f70f176a40a06d35d227824edf9d83a985068f97..31cb5e87b99a5ac61fef740bb8c9393c032ef75a 100755 --- a/packaging/make_install.sh +++ b/packaging/tools/make_install.sh @@ -169,6 +169,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : @@ -183,8 +184,9 @@ function install_bin() { [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : + [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : - ${csudo}cp -r ${binary_dir}/build/bin/tarbitrator ${install_main_dir}/bin || : + # ${csudo}cp -r ${binary_dir}/build/bin/tarbitrator ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || : @@ -197,6 +199,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : @@ -213,6 +216,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${clientName} ] || [ -x ${install_main_2_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || ${csudo}ln -s ${install_main_2_dir}/bin/${clientName} || : [ -x ${install_main_dir}/bin/${serverName} ] || [ -x ${install_main_2_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || ${csudo}ln -s ${install_main_2_dir}/bin/${serverName} || : [ -x ${install_main_dir}/bin/taosadapter ] || [ -x ${install_main_2_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || ${csudo}ln -s ${install_main_2_dir}/bin/taosadapter || : + [ -x ${install_main_dir}/bin/udfd ] || [ -x ${install_main_2_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || ${csudo}ln -s ${install_main_2_dir}/bin/udfd || : [ -x ${install_main_dir}/bin/taosdump ] || [ -x ${install_main_2_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || ln -s ${install_main_2_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] || [ -x ${install_main_2_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || ln -s ${install_main_2_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : fi @@ -368,15 +372,15 @@ function install_config() { if [ ! -f ${cfg_install_dir}/${configFile} ]; then ${csudo}mkdir -p ${cfg_install_dir} [ -f ${script_dir}/../cfg/${configFile} ] && - ${csudo}cp ${script_dir}/../cfg/${configFile} ${cfg_install_dir} || : + ${csudo}cp ${script_dir}/../cfg/${configFile} ${cfg_install_dir} ${csudo}chmod 644 ${cfg_install_dir}/${configFile} ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ - ${cfg_install_dir}/${configFile}.${verNumber} || : + ${cfg_install_dir}/${configFile}.${verNumber} ${csudo}ln -s ${cfg_install_dir}/${configFile} \ ${install_main_dir}/cfg/${configFile} else ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ - ${cfg_install_dir}/${configFile}.${verNumber} || : + ${cfg_install_dir}/${configFile}.${verNumber} fi } @@ -471,10 +475,10 @@ function install_service_on_sysvinit() { if ((${os_type} == 1)); then # ${csudo}cp -f ${script_dir}/../deb/${serverName} ${install_main_dir}/init.d - ${csudo}cp ${script_dir}/../deb/${serverName} ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/${serverName} || : + ${csudo}cp ${script_dir}/../deb/${serverName} ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/${serverName} elif ((${os_type} == 2)); then # ${csudo}cp -f ${script_dir}/../rpm/${serverName} ${install_main_dir}/init.d - ${csudo}cp ${script_dir}/../rpm/${serverName} ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/${serverName} || : + ${csudo}cp ${script_dir}/../rpm/${serverName} ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/${serverName} fi if ((${initd_mod} == 1)); then diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh new file mode 100644 index 0000000000000000000000000000000000000000..f0cc54a355d792c15c977d0276e3db891e9701f0 --- /dev/null +++ b/packaging/tools/makearbi.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +productName="TDengine" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/${productName}-enterprise-arbitrator-${version}" +else + install_dir="${release_dir}/${productName}-arbitrator-${version}" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh" +install_files="${script_dir}/install_arbi.sh" + +#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taosdef.h ${code_dir}/inc/taoserror.h" +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} + +if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh new file mode 100644 index 0000000000000000000000000000000000000000..a86f62a32cb69207baaf2cb4daccd28bedbe2921 --- /dev/null +++ b/packaging/tools/makeclient.sh @@ -0,0 +1,246 @@ +#!/bin/bash +# +# Generate tar.gz package for linux client in all os system +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 +dbName=$9 + +productName="TDengine" +clientName="taos" +configFile="taos.cfg" +tarName="taos.tar.gz" + +if [ "$osType" != "Darwin" ]; then + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/../..)" +else + script_dir=$(dirname $0) + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/../.. +fi + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' + +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/${productName}-enterprise-client-${version}" +else + install_dir="${release_dir}/${productName}-client-${version}" +fi + +# Directories and files. + +if [ "$osType" != "Darwin" ]; then + if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/${clientName} + bin_files="${build_dir}/bin/${clientName} \ + ${script_dir}/remove_client.sh" + else + bin_files="${build_dir}/bin/${clientName} \ + ${script_dir}/remove_client.sh \ + ${script_dir}/set_core.sh \ + ${script_dir}/get_client.sh" + fi + lib_files="${build_dir}/lib/libtaos.so.${version}" +else + bin_files="${build_dir}/bin/${clientName} ${script_dir}/remove_client.sh" + lib_files="${build_dir}/lib/libtaos.${version}.dylib" +fi + +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taosdef.h ${code_dir}/inc/taoserror.h" +if [ "$dbName" != "taos" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + +install_files="${script_dir}/install_client.sh" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* + +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + +cd ${install_dir} + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f ${tarName} * --remove-files || : +else + tar -zcv -f ${tarName} * || : + mv ${tarName} .. + rm -rf ./* + mv ../${tarName} . +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$osType" == "Darwin" ]; then + sed 's/osType=Linux/osType=Darwin/g' ${install_dir}/install_client.sh >>install_client_temp.sh + mv install_client_temp.sh ${install_dir}/install_client.sh +fi + +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_client.sh >>install_client_temp.sh + mv install_client_temp.sh ${install_dir}/install_client.sh +fi + +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client.sh >>install_client_temp.sh + mv install_client_temp.sh ${install_dir}/install_client.sh +fi +chmod a+x ${install_dir}/install_client.sh + +if [[ $productName == "TDengine" ]]; then + # Copy example code + mkdir -p ${install_dir}/examples + examples_dir="${top_dir}/examples" + cp -r ${examples_dir}/c ${install_dir}/examples + if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp -r ${examples_dir}/JDBC ${install_dir}/examples + cp -r ${examples_dir}/matlab ${install_dir}/examples + cp -r ${examples_dir}/python ${install_dir}/examples + cp -r ${examples_dir}/R ${install_dir}/examples + cp -r ${examples_dir}/go ${install_dir}/examples + cp -r ${examples_dir}/nodejs ${install_dir}/examples + cp -r ${examples_dir}/C# ${install_dir}/examples + mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../src/kit/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json + fi + + if [ "$verMode" == "cluster" ]; then + # Copy connector + connector_dir="${code_dir}/connector" + mkdir -p ${install_dir}/connector + if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ "$osType" != "Darwin" ]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + git clone --depth 1 https://github.com/taosdata/taos-connector-python ${install_dir}/connector/python + rm -rf ${install_dir}/connector/python/.git ||: +# cp -r ${connector_dir}/python ${install_dir}/connector + git clone --depth 1 https://github.com/taosdata/taos-connector-node ${install_dir}/connector/nodejs + rm -rf ${install_dir}/connector/nodejs/.git ||: + + git clone --depth 1 https://github.com/taosdata/taos-connector-dotnet ${install_dir}/connector/dotnet + rm -rf ${install_dir}/connector/dotnet/.git ||: +# cp -r ${connector_dir}/nodejs ${install_dir}/connector + git clone --depth 1 https://github.com/taosdata/libtaos-rs ${install_dir}/connector/rust + rm -rf ${install_dir}/connector/rust/.git ||: + fi + fi +fi +# Copy driver +mkdir -p ${install_dir}/driver +cp ${lib_files} ${install_dir}/driver + +# Copy connector +connector_dir="${code_dir}/connector" +mkdir -p ${install_dir}/connector + +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ "$osType" != "Darwin" ]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector + cp -r ${connector_dir}/nodejs ${install_dir}/connector +fi +# Copy release note +# cp ${script_dir}/release_note ${install_dir} + +# exit 1 + +cd ${release_dir} + +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} + +# if [ "$verMode" == "cluster" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# elif [ "$verMode" == "edge" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# else +# echo "unknow verMode, nor cluster or edge" +# exit 1 +# fi + +if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +else + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) || : + mv "$(basename ${pkg_name}).tar.gz" .. + rm -rf ./* + mv ../"$(basename ${pkg_name}).tar.gz" . +fi + +cd ${curr_dir} diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh new file mode 100644 index 0000000000000000000000000000000000000000..b4e60c13f5061f401bb4cd68382cb1fc3c0be6de --- /dev/null +++ b/packaging/tools/makepkg.sh @@ -0,0 +1,378 @@ +#!/bin/bash +# +# Generate tar.gz package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 +versionComp=$9 +dbName=${10} + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +productName="TDengine" +serverName="taosd" +clientName="taos" +configFile="taos.cfg" +tarName="taos.tar.gz" +dumpName="taosdump" +benchmarkName="taosBenchmark" +toolsName="taostools" +adapterName="taosadapter" +defaultPasswd="taosdata" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/${productName}-enterprise-server-${version}" +else + install_dir="${release_dir}/${productName}-server-${version}" +fi + +if [ -d ${top_dir}/src/kit/taos-tools/packaging/deb ]; then + cd ${top_dir}/src/kit/taos-tools/packaging/deb + [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" + + taostools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}') + taostools_install_dir="${release_dir}/${clientName}Tools-${taostools_ver}" + + cd ${curr_dir} +else + taostools_install_dir="${release_dir}/${clientName}Tools-${version}" +fi + +# Directories and files +if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/${serverName} + strip ${build_dir}/bin/${clientName} + # lite version doesn't include taosadapter, which will lead to no restful interface + bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark" + taostools_bin_files="" +else + + wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ + && echo "TDinsight.sh downloaded!" \ + || echo "failed to download TDinsight.sh" + # download TDinsight caches + orig_pwd=$(pwd) + tdinsight_caches="" + cd ${build_dir}/bin/ && \ + chmod +x TDinsight.sh + tdinsight_caches=$(./TDinsight.sh --download-only | xargs -i printf "${build_dir}/bin/{} ") + cd $orig_pwd + echo "TDinsight caches: $tdinsight_caches" + + taostools_bin_files=" ${build_dir}/bin/taosdump \ + ${build_dir}/bin/taosBenchmark \ + ${build_dir}/bin/TDinsight.sh \ + $tdinsight_caches" + + bin_files="${build_dir}/bin/${serverName} \ + ${build_dir}/bin/${clientName} \ + ${taostools_bin_files} \ + ${build_dir}/bin/taosadapter \ + ${build_dir}/bin/tarbitrator\ + ${script_dir}/remove.sh \ + ${script_dir}/set_core.sh \ + ${script_dir}/run_taosd_and_taosadapter.sh \ + ${script_dir}/startPre.sh \ + ${script_dir}/taosd-dump-cfg.gdb" +fi + +lib_files="${build_dir}/lib/libtaos.so.${version}" +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taosdef.h ${code_dir}/inc/taoserror.h" + +if [ "$dbName" != "taos" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + +install_files="${script_dir}/install.sh" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" + +init_file_deb=${script_dir}/../deb/taosd +init_file_rpm=${script_dir}/../rpm/taosd +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} + +if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then + cp ${compile_dir}/test/cfg/taosadapter.toml ${install_dir}/cfg || : +fi + +if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then + cp ${compile_dir}/test/cfg/taosadapter.service ${install_dir}/cfg || : +fi + +if [ -f "${cfg_dir}/${serverName}.service" ]; then + cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || : +fi + +if [ -f "${top_dir}/packaging/cfg/tarbitratord.service" ]; then + cp ${top_dir}/packaging/cfg/tarbitratord.service ${install_dir}/cfg || : +fi + +if [ -f "${top_dir}/packaging/cfg/nginxd.service" ]; then + cp ${top_dir}/packaging/cfg/nginxd.service ${install_dir}/cfg || : +fi + +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb +mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +if [ $adapterName != "taosadapter" ]; then + mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml + sed -i "s/path = \"\/var\/log\/taos\"/path = \"\/var\/log\/${productName}\"/g" ${install_dir}/cfg/$adapterName.toml + sed -i "s/password = \"taosdata\"/password = \"${defaultPasswd}\"/g" ${install_dir}/cfg/$adapterName.toml + + mv ${install_dir}/cfg/taosadapter.service ${install_dir}/cfg/$adapterName.service + sed -i "s/TDengine/${productName}/g" ${install_dir}/cfg/$adapterName.service + sed -i "s/taosAdapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service + sed -i "s/taosadapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service + + mv ${install_dir}/bin/taosadapter ${install_dir}/bin/${adapterName} + mv ${install_dir}/bin/run_taosd_and_taosadapter.sh ${install_dir}/bin/run_${serverName}_and_${adapterName}.sh + mv ${install_dir}/bin/taosd-dump-cfg.gdb ${install_dir}/bin/${serverName}-dump-cfg.gdb +fi + +if [ -n "${taostools_bin_files}" ]; then + mkdir -p ${taostools_install_dir} || echo -e "failed to create ${taostools_install_dir}" + mkdir -p ${taostools_install_dir}/bin \ + && cp ${taostools_bin_files} ${taostools_install_dir}/bin \ + && chmod a+x ${taostools_install_dir}/bin/* || : + + if [ -f ${top_dir}/src/kit/taos-tools/packaging/tools/install-taostools.sh ]; then + cp ${top_dir}/src/kit/taos-tools/packaging/tools/install-taostools.sh \ + ${taostools_install_dir}/ > /dev/null \ + && chmod a+x ${taostools_install_dir}/install-taostools.sh \ + || echo -e "failed to copy install-taostools.sh" + else + echo -e "install-taostools.sh not found" + fi + + if [ -f ${top_dir}/src/kit/taos-tools/packaging/tools/uninstall-taostools.sh ]; then + cp ${top_dir}/src/kit/taos-tools/packaging/tools/uninstall-taostools.sh \ + ${taostools_install_dir}/ > /dev/null \ + && chmod a+x ${taostools_install_dir}/uninstall-taostools.sh \ + || echo -e "failed to copy uninstall-taostools.sh" + else + echo -e "uninstall-taostools.sh not found" + fi + + if [ -f ${build_dir}/lib/libavro.so.23.0.0 ]; then + mkdir -p ${taostools_install_dir}/avro/{lib,lib/pkgconfig} || echo -e "failed to create ${taostools_install_dir}/avro" + cp ${build_dir}/lib/libavro.* ${taostools_install_dir}/avro/lib + cp ${build_dir}/lib/pkgconfig/avro-c.pc ${taostools_install_dir}/avro/lib/pkgconfig + fi +fi + +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh + mv remove_temp.sh ${install_dir}/bin/remove.sh + + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd + cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png + rm -rf ${install_dir}/nginxd/png + + if [ "$cpuType" == "aarch64" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ + elif [ "$cpuType" == "aarch32" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/ + fi + rm -rf ${install_dir}/nginxd/sbin/arm +fi + +cd ${install_dir} +tar -zcv -f ${tarName} * --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${tarName} error !!!" + exit $exitcode +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install.sh >>install_temp.sh + mv install_temp.sh ${install_dir}/install.sh +fi +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install.sh >>install_temp.sh + mv install_temp.sh ${install_dir}/install.sh +fi +chmod a+x ${install_dir}/install.sh + +if [[ $dbName == "taos" ]]; then + # Copy example code + mkdir -p ${install_dir}/examples + examples_dir="${top_dir}/examples" + cp -r ${examples_dir}/c ${install_dir}/examples + if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ -d ${examples_dir}/JDBC/connectionPools/target ]; then + rm -rf ${examples_dir}/JDBC/connectionPools/target + fi + if [ -d ${examples_dir}/JDBC/JDBCDemo/target ]; then + rm -rf ${examples_dir}/JDBC/JDBCDemo/target + fi + if [ -d ${examples_dir}/JDBC/mybatisplus-demo/target ]; then + rm -rf ${examples_dir}/JDBC/mybatisplus-demo/target + fi + if [ -d ${examples_dir}/JDBC/springbootdemo/target ]; then + rm -rf ${examples_dir}/JDBC/springbootdemo/target + fi + if [ -d ${examples_dir}/JDBC/SpringJdbcTemplate/target ]; then + rm -rf ${examples_dir}/JDBC/SpringJdbcTemplate/target + fi + if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then + rm -rf ${examples_dir}/JDBC/taosdemo/target + fi + + cp -r ${examples_dir}/JDBC ${install_dir}/examples + cp -r ${examples_dir}/matlab ${install_dir}/examples + cp -r ${examples_dir}/python ${install_dir}/examples + cp -r ${examples_dir}/R ${install_dir}/examples + cp -r ${examples_dir}/go ${install_dir}/examples + cp -r ${examples_dir}/nodejs ${install_dir}/examples + cp -r ${examples_dir}/C# ${install_dir}/examples + mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../src/kit/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json + fi +fi + +# Copy driver +mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt + +# Copy connector +if [ "$verMode" == "cluster" ]; then + connector_dir="${code_dir}/connector" + mkdir -p ${install_dir}/connector + if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + git clone --depth 1 https://github.com/taosdata/taos-connector-python ${install_dir}/connector/python + rm -rf ${install_dir}/connector/python/.git ||: + + git clone --depth 1 https://github.com/taosdata/taos-connector-node ${install_dir}/connector/nodejs + rm -rf ${install_dir}/connector/nodejs/.git ||: + + git clone --depth 1 https://github.com/taosdata/taos-connector-dotnet ${install_dir}/connector/dotnet + rm -rf ${install_dir}/connector/dotnet/.git ||: + + git clone --depth 1 https://github.com/taosdata/libtaos-rs ${install_dir}/connector/rust + rm -rf ${install_dir}/connector/rust/.git ||: + # cp -r ${connector_dir}/python ${install_dir}/connector + # cp -r ${connector_dir}/nodejs ${install_dir}/connector + fi +fi + +# Copy release note +cp ${script_dir}/release_note ${install_dir} + +# exit 1 + +cd ${release_dir} + +# install_dir has been distinguishes cluster from edege, so comments this code +pkg_name=${install_dir}-${osType}-${cpuType} + +taostools_pkg_name=${taostools_install_dir}-${osType}-${cpuType} + +# if [ "$verMode" == "cluster" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# elif [ "$verMode" == "edge" ]; then +# pkg_name=${install_dir}-${osType}-${cpuType} +# else +# echo "unknow verMode, nor cluster or edge" +# exit 1 +# fi + +if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then + pkg_name=${install_dir}-${verType}-${osType}-${cpuType} + taostools_pkg_name=${taostools_install_dir}-${verType}-${osType}-${cpuType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} + taostools_pkg_name=${taostools_pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" "$(basename ${install_dir})" --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +if [ -n "${taostools_bin_files}" ]; then + wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" + tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : + exitcode=$? + if [ "$exitcode" != "0" ]; then + echo "tar ${taostools_pkg_name}.tar.gz error !!!" + exit $exitcode + fi +fi + +cd ${curr_dir} diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh new file mode 100644 index 0000000000000000000000000000000000000000..59315dbe994c406310a43b62ced771c89ca27294 --- /dev/null +++ b/packaging/tools/post.sh @@ -0,0 +1,539 @@ +#!/bin/bash +# +# This file is used to install tdengine rpm package on centos systems. The operating system +# is required to use systemd to manage services at boot +# set -x + +iplist="" +serverFqdn="" + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) +# Dynamic directory +data_dir="/var/lib/taos" +log_dir="/var/log/taos" +data_link_dir="/usr/local/taos/data" +log_link_dir="/usr/local/taos/log" +install_main_dir="/usr/local/taos" + +# static directory +cfg_dir="/usr/local/taos/cfg" +bin_dir="/usr/local/taos/bin" +lib_dir="/usr/local/taos/driver" +init_d_dir="/usr/local/taos/init.d" +inc_dir="/usr/local/taos/include" + +cfg_install_dir="/etc/taos" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" + + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_taosadapter() { +# ${csudo}pkill -f taosadapter || : + pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function kill_taosd() { +# ${csudo}pkill -f taosd || : + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function install_include() { + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h|| : + ${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_lib() { + ${csudo}rm -f ${lib_link_dir}/libtaos* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos* || : + + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + + ${csudo}ldconfig +} + +function install_bin() { + # Remove links + ${csudo}rm -f ${bin_link_dir}/taos || : + ${csudo}rm -f ${bin_link_dir}/taosd || : + ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : + ${csudo}rm -f ${bin_link_dir}/taosdemo || : + ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/rmtaos || : + ${csudo}rm -f ${bin_link_dir}/set_core || : + + ${csudo}chmod 0555 ${bin_dir}/* + + #Make link + [ -x ${bin_dir}/taos ] && ${csudo}ln -s ${bin_dir}/taos ${bin_link_dir}/taos || : + [ -x ${bin_dir}/taosd ] && ${csudo}ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : + [ -x ${bin_dir}/taosadapter ] && ${csudo}ln -s ${bin_dir}/taosadapter ${bin_link_dir}/taosadapter || : + [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosdemo || : + [ -x ${bin_dir}/TDinsight.sh ] && ${csudo}ln -sf ${bin_dir}/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : + [ -x ${bin_dir}/taosdump ] && ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : + [ -x ${bin_dir}/set_core.sh ] && ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : +} + +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in "${arr[@]}" + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo}echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo}hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo}echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in "${arr[@]}" + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + +function install_taosadapter_config() { + if [ ! -f "${cfg_install_dir}/taosadapter.toml" ]; then + [ ! -d %{cfg_install_dir} ] && + ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/taosadapter.toml ] && ${csudo}cp ${cfg_dir}/taosadapter.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/taosadapter.toml ] && + ${csudo}chmod 644 ${cfg_install_dir}/taosadapter.toml + fi + + [ -f ${cfg_dir}/taosadapter.toml ] && + ${csudo}mv ${cfg_dir}/taosadapter.toml ${cfg_dir}/taosadapter.toml.new + + [ -f ${cfg_install_dir}/taosadapter.toml ] && + ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml ${cfg_dir} +} + +function install_config() { + if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then + ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/taos.cfg ] && ${csudo}cp ${cfg_dir}/taos.cfg ${cfg_install_dir} + ${csudo}chmod 644 ${cfg_install_dir}/* + fi + + # Save standard input to 6 and open / dev / TTY on standard input + exec 6<&0 0 ${email_file}" + break + #else + # read -p "Please enter the correct email address: " emailAddr + #fi + else + break + fi + done +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo}sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof taosd &> /dev/null; then + ${csudo}service taosd stop || : + fi + + if ((${initd_mod}==1)); then + ${csudo}chkconfig --del taosd || : + elif ((${initd_mod}==2)); then + ${csudo}insserv -r taosd || : + elif ((${initd_mod}==3)); then + ${csudo}update-rc.d -f taosd remove || : + fi + + ${csudo}rm -f ${service_config_dir}/taosd || : + + if $(which init &> /dev/null); then + ${csudo}init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + + sleep 1 + + # Install taosd service + ${csudo}cp %{init_d_dir}/taosd ${service_config_dir} && ${csudo}chmod a+x ${service_config_dir}/taosd + + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo}grep -q -F "$restart_config_str" /etc/inittab || ${csudo}bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo}chkconfig --add taosd || : + ${csudo}chkconfig --level 2345 taosd on || : + elif ((${initd_mod}==2)); then + ${csudo}insserv taosd || : + ${csudo}insserv -d taosd || : + elif ((${initd_mod}==3)); then + ${csudo}update-rc.d taosd defaults || : + fi +} + +function clean_service_on_systemd() { + taosd_service_config="${service_config_dir}/taosd.service" + + # taosd service already is stoped before install in preinst script + #if systemctl is-active --quiet taosd; then + # echo "TDengine is running, stopping it..." + # ${csudo}systemctl stop taosd &> /dev/null || echo &> /dev/null + #fi + ${csudo}systemctl disable taosd &> /dev/null || echo &> /dev/null + + ${csudo}rm -f ${taosd_service_config} +} + +# taos:2345:respawn:/etc/init.d/taosd start + +function install_service_on_systemd() { + clean_service_on_systemd + + taosd_service_config="${service_config_dir}/taosd.service" + + ${csudo}bash -c "echo '[Unit]' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'After=network-online.target' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'Wants=network-online.target' >> ${taosd_service_config}" + ${csudo}bash -c "echo >> ${taosd_service_config}" + ${csudo}bash -c "echo '[Service]' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'Type=simple' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'Restart=always' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" + ${csudo}bash -c "echo >> ${taosd_service_config}" + ${csudo}bash -c "echo '[Install]' >> ${taosd_service_config}" + ${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" + ${csudo}systemctl enable taosd +} + +function install_taosadapter_service() { + if ((${service_mod}==0)); then + [ -f ${script_dir}/../cfg/taosadapter.service ] &&\ + ${csudo}cp ${script_dir}/../cfg/taosadapter.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + fi +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # manual start taosd + kill_taosadapter + kill_taosd + fi +} + +function install_TDengine() { + echo -e "${GREEN}Start to install TDengine...${NC}" + + #install log and data dir , then ln to /usr/local/taos + ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} + ${csudo}mkdir -p ${data_dir} + + ${csudo}rm -rf ${log_link_dir} || : + ${csudo}rm -rf ${data_link_dir} || : + + ${csudo}ln -s ${log_dir} ${log_link_dir} || : + ${csudo}ln -s ${data_dir} ${data_link_dir} || : + + # Install include, lib, binary and service + install_include + install_lib + install_bin + install_config + install_taosadapter_config + install_taosadapter_service + install_service + + # Ask if to start the service + #echo + #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}systemctl start taosd${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}" + echo -e " : ${csudo}service taosd start ${RED} after${NC}" + else + echo -e "${GREEN_DARK}To start TDengine ${NC}: ./taosd${NC}" + fi + + + + if [ ! -z "$firstEp" ]; then + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" + echo + fi + echo + echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" +} + + +## ==============================Main program starts from here============================ +serverFqdn=$(hostname) +install_TDengine diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh new file mode 100644 index 0000000000000000000000000000000000000000..bc1727c4e470a25f99f248bcb370834415f375f5 --- /dev/null +++ b/packaging/tools/preun.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TSDB + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +data_link_dir="/usr/local/taos/data" +log_link_dir="/usr/local/taos/log" +cfg_link_dir="/usr/local/taos/cfg" + +service_config_dir="/etc/systemd/system" +taos_service_name="taosd" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_taosadapter() { + pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function kill_taosd() { + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function clean_service_on_systemd() { + taosadapter_service_config="${service_config_dir}/taosadapter.service" + if systemctl is-active --quiet taosadapter; then + echo "taosadapter is running, stopping it..." + ${csudo}systemctl stop taosadapter &> /dev/null || echo &> /dev/null + fi + + taosd_service_config="${service_config_dir}/${taos_service_name}.service" + + if systemctl is-active --quiet ${taos_service_name}; then + echo "TDengine taosd is running, stopping it..." + ${csudo}systemctl stop ${taos_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo}systemctl disable ${taos_service_name} &> /dev/null || echo &> /dev/null + + ${csudo}rm -f ${taosd_service_config} + + [ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config} + +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo}sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof taosd &> /dev/null; then + echo "TDengine taosd is running, stopping it..." + ${csudo}service taosd stop || : + fi + + if ((${initd_mod}==1)); then + ${csudo}chkconfig --del taosd || : + elif ((${initd_mod}==2)); then + ${csudo}insserv -r taosd || : + elif ((${initd_mod}==3)); then + ${csudo}update-rc.d -f taosd remove || : + fi + + ${csudo}rm -f ${service_config_dir}/taosd || : + + if $(which init &> /dev/null); then + ${csudo}init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop taosd + kill_taosadapter + kill_taosd + fi +} + +# Stop service and disable booting start. +clean_service + +# Remove all links +${csudo}rm -f ${bin_link_dir}/taos || : +${csudo}rm -f ${bin_link_dir}/taosd || : +${csudo}rm -f ${bin_link_dir}/taosadapter || : +${csudo}rm -f ${bin_link_dir}/taosBenchmark || : +${csudo}rm -f ${bin_link_dir}/taosdemo || : +${csudo}rm -f ${bin_link_dir}/set_core || : +${csudo}rm -f ${cfg_link_dir}/*.new || : +${csudo}rm -f ${inc_link_dir}/taos.h || : +${csudo}rm -f ${inc_link_dir}/taosdef.h || : +${csudo}rm -f ${inc_link_dir}/taoserror.h || : +${csudo}rm -f ${lib_link_dir}/libtaos.* || : +${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + +${csudo}rm -f ${log_link_dir} || : +${csudo}rm -f ${data_link_dir} || : + +if ((${service_mod}==2)); then + kill_taosadapter + kill_taosd +fi + +echo -e "${GREEN}TDengine is removed successfully!${NC}" diff --git a/packaging/tools/release_note b/packaging/tools/release_note new file mode 100644 index 0000000000000000000000000000000000000000..4a954b1f1249f3eb90fd7a021a05478ee5465b65 --- /dev/null +++ b/packaging/tools/release_note @@ -0,0 +1,136 @@ +taos-1.6.4.0 (Release on 2019-12-01) +Bug fixed: + 1.Look for possible causes of file corruption and fix them + 2.Encapsulate memory allocation functions to reduce the possibility of crashes + 3.Increase Arm64 compilation options + 4.Remove most of the warnings in the code + 5.Provide a variety of connector usage documents + 6.Network connection can be selected in udp and tcp + 7.Allow the maximum number of Tags to be 32 + 8.Bugs reported by the user + +taos-1.5.2.6 (Release on 2019-05-13) +Bug fixed: + - Nchar strings sometimes were wrongly truncated on Window + - Importing data from file throws an error of "invalid SQL" + +taos-1.5.2.5 (Release on 2019-05-13) +Bug fixed: + - Long timespan data import sometimes affects query result + - Synchronzation of cluster dnodes worked incorrectly when importing + +taos-1.5.2.4 (Release on 2019-05-10) +New Features: + - Optimized Windows client installation: now users don't need to copy taos.dll manually + - Changed the priority of taos.cfg and JDBC URL: parameters in JDCB URL now has a higher priority than parameters in taos.cfg +Bug fixed: + - Expired data files were not deleted corrected + - Occasionally importing returned "affected rows" which larger than 0, but 0 row was actually written into db + - Commit log is occupied by too many import-to-file requests, which blocked further data importing + - Cloud service shows a wrong number of available days with current balance + - Other minor issues + +taos-1.5.1 (Release on 2019-04-09) +New Features: + - Maximum number of rows returned by "top/bottom" methods increased from 20 to 100 + - Improved the performance of "first/last" methods + - Increased system stability +Bug fixed: + - Connection failure when query on huge STables through TPC + - The primary timestamp is occasionally returned as NULL in some queries + - Operation failure when updating a tag value to NULL + - Stream calculation couldn't start at certain occasions + +taos-1.5.0 (Release on 2019-03-11) +New Features: + - New syntax to automatically create tables when inserting values into non-existing tables + - New syntax "slimit/soffset" to pagenate groups in a query result set + - Support "top/bottom" queries on a supertable + - High performance statistic aggregation function "apercentile" + - Remove "first_t/last_t" functions; improve the performance of "first/last" function + - Add pre-aggregation for bool type values + - Supports fixed-length streaming computation, i.e. users may define an end time for a stream + - New JAVA API for SQL subscription, supports table/supertable/SQL query subscription +Bug fixed: + - Data file broken issue when frequently using "import" + - Using "spread" on a super table may return negative values + - RPC bug that random network packets might cause the RPC module to crush + +taos-1.4.15 (Released on 2019-01-23) +New Features: + - JDBC Driver now supports configuring timezone, locale, cfgdir in JDBC url + - A new API is added to validate if a table creation sql statement is correct in syntax without actually creating that table +Bugs Fixed: + - "select last(*) from STable" sometimest returned incorrect number of rows + - JDBC driver method ResultSetMetaData.getColumnClassName() returned wrong values. + - Web shell automatically changed query string to lower case + +taos-1.4.14 (Released on 2018-12-22) +New Features: + - C Driver support for integration with Python + - JDBC Driver support for integration with R and MATLAB + +taos-1.4.13 (Released on 2018-12-14) +Bugs Fixed: + - Clients failed to connect to server due to unexpected and invalid packets recieved by the server. +Features Added: + - Add support to HikariCP in TSDB JDBC driver. + +taos-1.4.12 (Released on 2018-12-08) +Bugs Fixed: + - Querying data while inserting into the database might return incomplete resultsets. +Features Added: + - A new python driver is added. + - Increased system stability. + - Changed meaning of database configuration paramerter 'ablocks'. 'ablocks' used to refer to the number of total cache blocks in memory, now it refers to average number of cache blocks for each table in memory. + +taos-1.4.11 (Released on 2018-11-23) +Bugs Fixed: + - Thread memory leaking during high-frequency committing. + - Master dnode selection failure caused by accidental network issues. +Features Added: + - Change keyword "metrics" to "stables", i.e. supertables; the previous query "show metrics" is now changed to "show stables". + - Add an error message mechanism in C# driver. An error with message "Failed to connect to server" is thrown when fetching data experienced a network connection interruption during data transmitting. + +taos-1.4.10 (Released on 2018-11-13) +Bugs Fixed: + - Taosdump failed while exporting extremely large datasets to a .sql file. + - Commit status did not change correctly if the last commit was triggered by commit threshold time (ctime) and no more new data was written to DB during the next ctime period. +Features Added: + - Support importing historical data from Telegraf interface. + - Support MyBatis framework in TSDB JDBC Driver. + - Change result set row indexing in JDBC Driver. Result set row indexes now starts from 1 instead of 0. + +taos-1.4.9 (Released on 2018-11-02) +Bugs Fixed: + - Dumping data using UTF-8 format in client shell failed. + - Tag query failed using C# Driver. + - Committing data to disk failed if DB files were corrupted. + - Continuously pressing Ctrl+c in client shell for multiple times produced a segmentation fault. +Features Added: + - Changed the display pattern in shell for taosdump. + - Add a check to the status of an existing resultset before firing a new query in a single JDBC connection. A connection can only have a single open resultset, and the resultset must be closed before one can execute new queries. + + +taos-1.4.7 (Released on 2018-10-25) +Bug Fixed: + - UTF-8 encoding in JDBC Driver did not give the correct Chinese characters. + - Fix crash error when where clause is too long. +Features Added: + - Add check on database properties, force ablocks to be at least (4 * tables) in a vnode. + - Check if pVgroup is empty in sdb. + +taos-1.4.6 (Released on 2018-10-21) +Bug Fixed: + - Fix wrong symbol addition while export csv file. +Features Added: + - Update grafana plugins. + - Update python drivers. + - Add error code explanation in JDBC Driver. + - Prohibit login while the version of server and client are not match. + +taos-1.4.5 (Released on 2018-10-17) +Bug Fixed: + - Fix HTTP request truncation bug in Telegraf interface. +Features Added: + - Support nchar and null object in JDBC Driver. diff --git a/packaging/remove.sh b/packaging/tools/remove.sh similarity index 100% rename from packaging/remove.sh rename to packaging/tools/remove.sh diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh new file mode 100644 index 0000000000000000000000000000000000000000..9b8e49d511d2e93a967b1397ce9a4de670422466 --- /dev/null +++ b/packaging/tools/remove_arbi.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TDengine's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tarbitrator" +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo}rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosdef.h || : + ${csudo}rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo}rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo}systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo}systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo}rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "TDengine's tarbitrator is running, stopping it..." + ${csudo}service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo}rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo}init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo}rm -rf ${install_main_dir} + +echo -e "${GREEN}TDengine's arbitrator is removed successfully!${NC}" diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh new file mode 100644 index 0000000000000000000000000000000000000000..9e22f894ac99c9930a35ddeb1c4994a469da5a98 --- /dev/null +++ b/packaging/tools/remove_client.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Script to stop the client and uninstall database, but retain the config and log files. +set -e +# set -x + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +installDir="/usr/local/taos" +clientName="taos" +uninstallScript="rmtaos" + +#install main path +install_main_dir=${installDir} + +log_link_dir=${installDir}/log +cfg_link_dir=${installDir}/cfg +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +function kill_client() { + if [ -n "$(pidof ${clientName})" ]; then + ${csudo}kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo}rm -f ${bin_link_dir}/${clientName} || : + ${csudo}rm -f ${bin_link_dir}/taosdemo || : + ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo}rm -rf ${v15_java_app_dir} || : +} + +function clean_header() { + # Remove link + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosdef.h || : + ${csudo}rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo}rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo}rm -rf ${log_link_dir} || : +} + +# Stop client. +kill_client +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config + +${csudo}rm -rf ${install_main_dir} + +echo -e "${GREEN}TDengine client is removed successfully!${NC}" +echo diff --git a/packaging/tools/repair_link.sh b/packaging/tools/repair_link.sh new file mode 100644 index 0000000000000000000000000000000000000000..8507b639ba80dc062b7c859b4b166a637d29c031 --- /dev/null +++ b/packaging/tools/repair_link.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# This script is used to repaire links when you what to move TDengine +# data to other places and to access data. + +# Read link path +read -p "Please enter link directory such as /var/lib/taos/tsdb: " linkDir + +while true; do + if [ ! -d $linkDir ]; then + read -p "Paht not exists, please enter the correct link path:" linkDir + continue + fi + break +done + +declare -A dirHash + +for linkFile in $(find -L $linkDir -xtype l); do + targetFile=$(readlink -f $linkFile) + echo "targetFile: ${targetFile}" + # TODO : Extract directory part and basename part + dirName=$(dirname $(dirname ${targetFile})) + baseName=$(basename $(dirname ${targetFile}))/$(basename ${targetFile}) + + # TODO : + newDir="${dirHash["$dirName"]}" + if [ -z "${dirHash["$dirName"]}" ]; then + read -p "Please enter the directory to replace ${dirName}:" newDir + + read -p "Do you want to replcace all[y/N]?" replcaceAll + if [[ ( "${replcaceAll}" == "y") || ( "${replcaceAll}" == "Y") ]]; then + dirHash["$dirName"]="$newDir" + fi + fi + + # Replcace the file + ln -sf "${newDir}/${baseName}" "${linkFile}" +done diff --git a/packaging/tools/run_taosd_and_taosadapter.sh b/packaging/tools/run_taosd_and_taosadapter.sh new file mode 100644 index 0000000000000000000000000000000000000000..3dfab87d72896f7b51bcabd5e001a8fb07550255 --- /dev/null +++ b/packaging/tools/run_taosd_and_taosadapter.sh @@ -0,0 +1,3 @@ +#!/bin/bash +[[ -x /usr/bin/taosadapter ]] && /usr/bin/taosadapter & +taosd diff --git a/packaging/tools/set_core.sh b/packaging/tools/set_core.sh new file mode 100644 index 0000000000000000000000000000000000000000..41e8aebec3b878f073a2a078c01bb0ef978b2f4e --- /dev/null +++ b/packaging/tools/set_core.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# This file is used to set config for core when taosd crash + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +# set -e +# set -x +corePath=$1 + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo " +fi + +if [[ ! -n ${corePath} ]]; then + echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:" + read corePath + while true; do + if [[ ! -z "$corePath" ]]; then + break + else + read -p "Please enter a file directory to save the coredump file:" corePath + fi + done +fi + +ulimit -c unlimited +${csudo}sed -i '/ulimit -c unlimited/d' /etc/profile ||: +${csudo}sed -i '$a\ulimit -c unlimited' /etc/profile ||: +source /etc/profile + +${csudo}mkdir -p ${corePath} ||: +${csudo}sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||: +${csudo}echo "${corePath}/core-%e-%p" | ${csudo}tee /proc/sys/kernel/core_pattern ||: diff --git a/packaging/tools/startPre.sh b/packaging/tools/startPre.sh new file mode 100644 index 0000000000000000000000000000000000000000..341e88ab2eaccd2eb8d1b985194ed687cd7c4479 --- /dev/null +++ b/packaging/tools/startPre.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# if enable core dump, set start count to 3, disable core dump, set start count to 20. +# set -e +# set -x + +serverName="taosd" +logDir="/var/log/taos" + +taosd=/etc/systemd/system/${serverName}.service +line=$(grep StartLimitBurst ${taosd}) +num=${line##*=} +#echo "burst num: ${num}" + +startSeqFile=${logDir}/.startSeq +recordFile=${logDir}/.startRecord + +startSeq=0 + +if [[ ! -e ${startSeqFile} ]]; then + startSeq=0 +else + startSeq=$(cat ${startSeqFile}) +fi + +nextSeq=$(expr $startSeq + 1) +echo "${nextSeq}" >${startSeqFile} + +curTime=$(date "+%Y-%m-%d %H:%M:%S") +echo "startSeq:${startSeq} startPre.sh exec ${curTime}, burstCnt:${num}" >>${recordFile} + +coreFlag=$(ulimit -c) +echo "coreFlag: ${coreFlag}" >>${recordFile} + +if [ ${coreFlag} = "0" ]; then + #echo "core is 0" + if [ ${num} != "20" ]; then + sed -i "s/^.*StartLimitBurst.*$/StartLimitBurst=20/" ${taosd} + systemctl daemon-reload + echo "modify burst count from ${num} to 20" >>${recordFile} + fi +fi + +if [ ${coreFlag} = "unlimited" ]; then + #echo "core is unlimited" + if [ ${num} != "3" ]; then + sed -i "s/^.*StartLimitBurst.*$/StartLimitBurst=3/" ${taosd} + systemctl daemon-reload + echo "modify burst count from ${num} to 3" >>${recordFile} + fi +fi diff --git a/packaging/tools/taosd-dump-cfg.gdb b/packaging/tools/taosd-dump-cfg.gdb new file mode 100644 index 0000000000000000000000000000000000000000..a7b143221d22c59236be701e68e4ff3a027212e8 --- /dev/null +++ b/packaging/tools/taosd-dump-cfg.gdb @@ -0,0 +1,144 @@ +# Usage: +# sudo gdb -x ./taosd-dump-cfg.gdb + +define attach_pidof + if $argc != 1 + help attach_pidof + else + shell echo -e "\ +set \$PID = "$(echo $(pidof $arg0) 0 | cut -d " " -f 1)"\n\ +if \$PID > 0\n\ + attach "$(pidof -s $arg0)"\n\ +else\n\ + print \"Process '"$arg0"' not found\"\n\ +end" > /tmp/gdb.pidof + source /tmp/gdb.pidof + end +end + +document attach_pidof +Attach to process by name +Usage: attach_pidof PROG_NAME +end + +set $TAOS_CFG_VTYPE_INT8 = 0 +set $TAOS_CFG_VTYPE_INT16 = 1 +set $TAOS_CFG_VTYPE_INT32 = 2 +set $TAOS_CFG_VTYPE_FLOAT = 3 +set $TAOS_CFG_VTYPE_STRING = 4 +set $TAOS_CFG_VTYPE_IPSTR = 5 +set $TAOS_CFG_VTYPE_DIRECTORY = 6 + +set $TSDB_CFG_CTYPE_B_CONFIG = 1U +set $TSDB_CFG_CTYPE_B_SHOW = 2U +set $TSDB_CFG_CTYPE_B_LOG = 4U +set $TSDB_CFG_CTYPE_B_CLIENT = 8U +set $TSDB_CFG_CTYPE_B_OPTION = 16U +set $TSDB_CFG_CTYPE_B_NOT_PRINT = 32U + +set $TSDB_CFG_PRINT_LEN = 53 + +define print_blank + if $argc == 1 + set $blank_len = $arg0 + while $blank_len > 0 + printf "%s", " " + set $blank_len = $blank_len - 1 + end + end +end + +define dump_cfg + if $argc != 1 + help dump_cfg + else + set $blen = $TSDB_CFG_PRINT_LEN - (int)strlen($arg0.option) + if $blen < 0 + $blen = 0 + end + #printf "%s: %d\n", "******blen: ", $blen + printf "%s: ", $arg0.option + print_blank $blen + + if $arg0.valType == $TAOS_CFG_VTYPE_INT8 + printf "%d\n", *((int8_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_INT16 + printf "%d\n", *((int16_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_INT32 + printf "%d\n", *((int32_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_FLOAT + printf "%f\n", *((float *) $arg0.ptr) + else + printf "%s\n", $arg0.ptr + end + end + end + end + end +end + +document dump_cfg +Dump a cfg entry +Usage: dump_cfg cfg +end + +set pagination off + +attach_pidof taosd + +set $idx=0 +#print tsGlobalConfigNum +#set $end=$1 +set $end=tsGlobalConfigNum + +p "*=*=*=*=*=*=*=*=*= taos global config:" +#while ($idx .lt. $end) +while ($idx < $end) + # print tsGlobalConfig[$idx].option + set $cfg = tsGlobalConfig[$idx] + set $tsce = tscEmbedded +# p "1" + if ($tsce == 0) + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_CLIENT) + end + else + if $cfg.cfgType & $TSDB_CFG_CTYPE_B_NOT_PRINT + else + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_SHOW) + else + dump_cfg $cfg + end + end + end + + set $idx=$idx+1 +end + +set $idx=0 + +p "*=*=*=*=*=*=*=*=*= taos local config:" +while ($idx < $end) + set $cfg = tsGlobalConfig[$idx] + set $tsce = tscEmbedded + if ($tsce == 0) + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_CLIENT) + end + else + if $cfg.cfgType & $TSDB_CFG_CTYPE_B_NOT_PRINT + else + if ($cfg.cfgType & $TSDB_CFG_CTYPE_B_SHOW) + else + dump_cfg $cfg + end + end + end + + set $idx=$idx+1 +end + +detach + +quit diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index ae27e611cb4fada8b6b3c8255f6595f80e50f5ce..f0c9dcd67dd8e3b05775003221ddf86681da37ab 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -63,7 +63,7 @@ typedef struct SStmtBindInfo { int8_t tbType; bool tagsCached; void* boundTags; - char tbName[TSDB_TABLE_FNAME_LEN];; + char tbName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN]; char stbFName[TSDB_TABLE_FNAME_LEN]; SName sname; @@ -71,7 +71,6 @@ typedef struct SStmtBindInfo { typedef struct SStmtExecInfo { int32_t affectedRows; - bool emptyRes; SRequestObj* pRequest; SHashObj* pVgHash; SHashObj* pBlockHash; @@ -87,7 +86,6 @@ typedef struct SStmtSQLInfo { char* sqlStr; int32_t sqlLen; SArray* nodeList; - SQueryPlan* pQueryPlan; SStmtQueryResInfo queryRes; bool autoCreateTbl; } SStmtSQLInfo; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 819c50275b737238f11b74b19c3fb441a5d38aaf..c82cce9af3916aa13439949df2c62831ad16fc59 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -91,7 +91,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; - rpcInit.sessions = tsMaxConnections; + rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 3e71714f21cdad052a1608b19f91f002ba02189c..fc909cb0a355be620283fd461ac8634e0609db8e 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -54,29 +54,29 @@ typedef enum { } ESchemaAction; typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SArray *tags; - SArray *fields; + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray *tags; + SArray *fields; } SCreateSTableActionInfo; typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SSmlKv * field; + char sTableName[TSDB_TABLE_NAME_LEN]; + SSmlKv *field; } SAlterSTableActionInfo; typedef struct { - ESchemaAction action; + ESchemaAction action; union { SCreateSTableActionInfo createSTable; - SAlterSTableActionInfo alterSTable; + SAlterSTableActionInfo alterSTable; }; } SSchemaAction; typedef struct { - const char* measure; - const char* tags; - const char* cols; - const char* timestamp; + const char *measure; + const char *tags; + const char *cols; + const char *timestamp; int32_t measureLen; int32_t measureTagsLen; @@ -103,7 +103,7 @@ typedef struct { SHashObj *tagHash; // elements are SArray *cols; - SHashObj *fieldHash; + SHashObj *colHash; STableMeta *tableMeta; } SSmlSTableMeta; @@ -180,6 +180,7 @@ static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashOb } static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const char *msg2) { + memset(pBuf->buf, 0 , pBuf->len); if(msg1) strncat(pBuf->buf, msg1, pBuf->len); int32_t left = pBuf->len - strlen(pBuf->buf); if(left > 2 && msg2) { @@ -189,47 +190,39 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } -static int32_t smlGenerateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], +static int32_t smlGenerateSchemaAction(SSchema* colField, SHashObj* colHash, SSmlKv* kv, bool isTag, SSchemaAction* action, bool* actionNeeded, SSmlHandle* info) { -// char fieldName[TSDB_COL_NAME_LEN] = {0}; -// strcpy(fieldName, pointColField->name); -// -// size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName)); -// if (pDbIndex) { -// SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); -// assert(strcasecmp(dbAttr->name, pointColField->name) == 0); -// if (pointColField->type != dbAttr->type) { -// uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, -// pointColField->type, dbAttr->type); -// return TSDB_CODE_TSC_INVALID_VALUE; -// } -// -// if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { -// if (isTag) { -// action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; -// } else { -// action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; -// } -// memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); -// memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); -// action->alterSTable.field = pointColField; -// *actionNeeded = true; -// } -// } else { -// if (isTag) { -// action->action = SCHEMA_ACTION_ADD_TAG; -// } else { -// action->action = SCHEMA_ACTION_ADD_COLUMN; -// } -// memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); -// memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); -// action->alterSTable.field = pointColField; -// *actionNeeded = true; -// } -// if (*actionNeeded) { -// uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldName, -// action->action); -// } + uint16_t *index = (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen); + if (index) { + if (colField[*index].type != kv->type) { + uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, kv->key, + colField[*index].type, kv->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || + (colField[*index].type == TSDB_DATA_TYPE_NCHAR &&((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { + if (isTag) { + action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + action->alterSTable.field = kv; + *actionNeeded = true; + } + } else { + if (isTag) { + action->action = SCHEMA_ACTION_ADD_TAG; + } else { + action->action = SCHEMA_ACTION_ADD_COLUMN; + } + action->alterSTable.field = kv; + *actionNeeded = true; + } + if (*actionNeeded) { + uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, colField->name, + action->action); + } return 0; } @@ -238,9 +231,9 @@ static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSi char tname[TSDB_TABLE_NAME_LEN] = {0}; memcpy(tname, field->key, field->keyLen); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = field->valueLen; // todo + int32_t bytes = field->length; // todo int out = snprintf(buf, bufSize,"`%s` %s(%d)", - tname,tDataTypes[field->type].name, bytes); + tname, tDataTypes[field->type].name, bytes); *outBytes = out; } else { int out = snprintf(buf, bufSize, "`%s` %s", tname, tDataTypes[type].name); @@ -259,7 +252,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { uDebug("SML:0x%"PRIx64" apply schema action. action: %d", info->id, action->action); switch (action->action) { case SCHEMA_ACTION_ADD_COLUMN: { - int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); + int n = sprintf(result, "alter stable `%s` add column ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); @@ -282,7 +275,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { break; } case SCHEMA_ACTION_ADD_TAG: { - int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); + int n = sprintf(result, "alter stable `%s` add tag ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery @@ -306,7 +299,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { break; } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { - int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); + int n = sprintf(result, "alter stable `%s` modify column ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery @@ -329,7 +322,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { - int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); + int n = sprintf(result, "alter stable `%s` modify tag ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery @@ -408,6 +401,25 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { return code; } +static int32_t smlProcessSchemaAction(SSmlHandle* info, SSchema* schemaField, SHashObj* schemaHash, SArray *cols, SSchemaAction* action, bool isTag){ + int32_t code = TSDB_CODE_SUCCESS; + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + SSmlKv* kv = (SSmlKv*)taosArrayGetP(cols, j); + bool actionNeeded = false; + code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, &actionNeeded, info); + if(code != TSDB_CODE_SUCCESS){ + return code; + } + if (actionNeeded) { + code = smlApplySchemaAction(info, action); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t smlModifyDBSchemas(SSmlHandle* info) { int32_t code = 0; @@ -427,27 +439,53 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) { - SSchemaAction schemaAction = { SCHEMA_ACTION_CREATE_STABLE, 0}; + SSchemaAction schemaAction; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = sTableData->tags; schemaAction.createSTable.fields = sTableData->cols; code = smlApplySchemaAction(info, &schemaAction); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" smlApplySchemaAction failed. can not create %s", info->id, schemaAction.createSTable.sTableName); return code; } + info->cost.numOfCreateSTables++; + }else if (code == TSDB_CODE_SUCCESS) { + SHashObj *hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + for(uint16_t i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++){ + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + } - code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); - if (code != 0) { - uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, schemaAction.createSTable.sTableName); + SSchemaAction schemaAction; + memset(&schemaAction, 0, sizeof(SSchemaAction)); + memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &schemaAction, true); + if (code != TSDB_CODE_SUCCESS) { + taosHashCleanup(hashTmp); + return code; + } + + taosHashClear(hashTmp); + for(uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++){ + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + } + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &schemaAction, false); + taosHashCleanup(hashTmp); + if (code != TSDB_CODE_SUCCESS) { return code; } - info->cost.numOfCreateSTables++; - }else if (code == TSDB_CODE_SUCCESS) { } else { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); return code; } + if(pTableMeta) taosMemoryFree(pTableMeta); + + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, (char*)superTable); + return code; + } sTableData->tableMeta = pTableMeta; tableMetaSml = (SSmlSTableMeta**)taosHashIterate(info->superTables, tableMetaSml); @@ -507,7 +545,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ const char *pVal = kvVal->value; - int32_t len = kvVal->valueLen; + int32_t len = kvVal->length; char *endptr = NULL; double result = strtod(pVal, &endptr); if(pVal == endptr){ @@ -591,7 +629,7 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ static bool smlParseBool(SSmlKv *kvVal) { const char *pVal = kvVal->value; - int32_t len = kvVal->valueLen; + int32_t len = kvVal->length; if ((len == 1) && pVal[0] == 't') { kvVal->i = true; return true; @@ -690,7 +728,7 @@ static int8_t smlGetTsTypeByLen(int32_t len) { if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { return TSDB_TIME_PRECISION_SECONDS; } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI_DIGITS; + return TSDB_TIME_PRECISION_MILLI; } else { return -1; } @@ -756,9 +794,12 @@ static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArra int64_t ts = 0; if(info->protocol == TSDB_SML_LINE_PROTOCOL){ ts = smlParseInfluxTime(info, data, len); - }else{ + }else if(info->protocol == TSDB_SML_TELNET_PROTOCOL){ ts = smlParseOpenTsdbTime(info, data, len); + }else{ + ASSERT(0); } + if(ts == -1) return TSDB_CODE_TSC_INVALID_TIME_STAMP; // add ts to @@ -778,18 +819,16 @@ static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArra static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { //binary - if (smlIsBinary(pVal->value, pVal->valueLen)) { + if (smlIsBinary(pVal->value, pVal->length)) { pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->valueLen -= BINARY_ADD_LEN; - pVal->length = pVal->valueLen; + pVal->length -= BINARY_ADD_LEN; pVal->value += (BINARY_ADD_LEN - 1); return true; } //nchar - if (smlIsNchar(pVal->value, pVal->valueLen)) { + if (smlIsNchar(pVal->value, pVal->length)) { pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->valueLen -= NCHAR_ADD_LEN; - pVal->length = pVal->valueLen; + pVal->length -= NCHAR_ADD_LEN; pVal->value += (NCHAR_ADD_LEN - 1); return true; } @@ -953,8 +992,8 @@ static int32_t smlParseTelnetTags(const char* data, int32_t len, SArray *cols, S kv->key = key; kv->keyLen = keyLen; kv->value = value; - kv->valueLen = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; + kv->length = valueLen; + kv->type = TSDB_DATA_TYPE_NCHAR; //todo if(cols) taosArrayPush(cols, &kv); } @@ -1002,7 +1041,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable kv->key = VALUE; kv->keyLen = VALUE_LEN; kv->value = value; - kv->valueLen = valueLen; + kv->length = valueLen; if(!smlParseValue(kv, &info->msgBuf) || kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_NCHAR || kv->type == TSDB_DATA_TYPE_BOOL){ return TSDB_CODE_SML_INVALID_DATA; @@ -1028,7 +1067,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is kv->key = TAG; kv->keyLen = TAG_LEN; kv->value = TAG; - kv->valueLen = TAG_LEN; + kv->length = TAG_LEN; kv->type = TSDB_DATA_TYPE_NCHAR; if(cols) taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; @@ -1086,7 +1125,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is kv->key = key; kv->keyLen = keyLen; kv->value = value; - kv->valueLen = valueLen; + kv->length = valueLen; if(isTag){ kv->type = TSDB_DATA_TYPE_NCHAR; }else{ @@ -1142,73 +1181,40 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is // return TSDB_CODE_SUCCESS; //} -static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SSmlMsgBuf *msg){ - if(tags){ - for (int i = 0; i < taosArrayGetSize(tags); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(tags, i); - ASSERT(kv->type == TSDB_DATA_TYPE_NCHAR); - - uint8_t *index = (uint8_t *)taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); - if(index){ - SSmlKv **value = (SSmlKv **)taosArrayGet(tableMeta->tags, *index); - ASSERT((*value)->type == TSDB_DATA_TYPE_NCHAR); - if(kv->valueLen > (*value)->valueLen){ // tags type is nchar - *value = kv; - } - }else{ - size_t tmp = taosArrayGetSize(tableMeta->tags); - ASSERT(tmp <= UINT8_MAX); - uint8_t size = tmp; - taosArrayPush(tableMeta->tags, &kv); - taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &size, CHAR_BYTES); - } - } - } - - if(cols){ - for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); +static bool smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg){ + for (int i = 0; i < taosArrayGetSize(cols); ++i) { //jump timestamp + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - int16_t *index = (int16_t *)taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); - if(index){ - SSmlKv **value = (SSmlKv **)taosArrayGet(tableMeta->cols, *index); - if(kv->type != (*value)->type){ - smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return false; - }else{ - if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger - if(kv->valueLen > (*value)->valueLen){ - *value = kv; - } + int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); + if(index){ + SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index); + if(kv->type != (*value)->type){ + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return false; + }else{ + if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger + if(kv->length > (*value)->length){ + *value = kv; } } - }else{ - size_t tmp = taosArrayGetSize(tableMeta->cols); - ASSERT(tmp <= INT16_MAX); - int16_t size = tmp; - taosArrayPush(tableMeta->cols, &kv); - taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &size, SHORT_BYTES); } + }else{ + size_t tmp = taosArrayGetSize(metaArray); + ASSERT(tmp <= INT16_MAX); + int16_t size = tmp; + taosArrayPush(metaArray, &kv); + taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); } } + return true; } -static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ - if(tags){ - for (uint8_t i = 0; i < taosArrayGetSize(tags); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(tags, i); - taosArrayPush(tableMeta->tags, &kv); - taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &i, CHAR_BYTES); - } - } - - if(cols){ - for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosArrayPush(tableMeta->cols, &kv); - taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &i, SHORT_BYTES); - } +static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols){ + for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + taosArrayPush(metaArray, &kv); + taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); } } @@ -1236,12 +1242,16 @@ cleanup: return NULL; } -static void smlDestroyTableInfo(SSmlTableInfo *tag, bool format){ - if(format){ +static void smlDestroyTableInfo(SSmlHandle* info, SSmlTableInfo *tag){ + if(info->dataFormat){ for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){ SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i); for (int j = 0; j < taosArrayGetSize(kvArray); ++j) { - void *p = taosArrayGetP(kvArray, j); + SSmlKv *p = (SSmlKv *)taosArrayGetP(kvArray, j); + if(info->protocol == TSDB_SML_JSON_PROTOCOL && + (p->type == TSDB_DATA_TYPE_NCHAR || p->type == TSDB_DATA_TYPE_BINARY)){ + taosMemoryFree((void*)p->value); + } taosMemoryFree(p); } taosArrayDestroy(kvArray); @@ -1257,6 +1267,19 @@ static void smlDestroyTableInfo(SSmlTableInfo *tag, bool format){ taosHashCleanup(kvHash); } } + for(size_t i = 0; i < taosArrayGetSize(tag->tags); i++){ + SSmlKv *p = (SSmlKv *)taosArrayGetP(tag->tags, i); + if(info->protocol == TSDB_SML_JSON_PROTOCOL){ + taosMemoryFree((void*)p->key); + if(p->type == TSDB_DATA_TYPE_NCHAR || p->type == TSDB_DATA_TYPE_BINARY){ + taosMemoryFree((void*)p->value); + } + } + taosMemoryFree(p); + } + if(info->protocol == TSDB_SML_JSON_PROTOCOL && tag->sTableName){ + taosMemoryFree((void*)tag->sTableName); + } taosArrayDestroy(tag->cols); taosArrayDestroy(tag->tags); taosMemoryFree(tag); @@ -1293,8 +1316,8 @@ static SSmlSTableMeta* smlBuildSTableMeta(){ goto cleanup; } - meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->fieldHash == NULL) { + meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->colHash == NULL) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; } @@ -1319,7 +1342,7 @@ cleanup: static void smlDestroySTableMeta(SSmlSTableMeta *meta){ taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->fieldHash); + taosHashCleanup(meta->colHash); taosArrayDestroy(meta->tags); taosArrayDestroy(meta->cols); taosMemoryFree(meta->tableMeta); @@ -1341,7 +1364,7 @@ static void smlDestroyInfo(SSmlHandle* info){ // destroy info->childTables void** p1 = (void**)taosHashIterate(info->childTables, NULL); while (p1) { - smlDestroyTableInfo((SSmlTableInfo*)(*p1), info->dataFormat); + smlDestroyTableInfo(info, (SSmlTableInfo*)(*p1)); p1 = (void**)taosHashIterate(info->childTables, p1); } taosHashCleanup(info->childTables); @@ -1357,7 +1380,9 @@ static void smlDestroyInfo(SSmlHandle* info){ // destroy info->pVgHash taosHashCleanup(info->pVgHash); taosHashCleanup(info->dumplicateKey); - + if(!info->dataFormat){ + taosArrayDestroy(info->colsContainer); + } taosMemoryFreeClear(info); } @@ -1682,8 +1707,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char* typeStr, cJSON *value) { return TSDB_CODE_TSC_INVALID_JSON_TYPE; } pVal->length = (int16_t)strlen(value->valuestring); - pVal->valueLen = pVal->length; - return smlJsonCreateSring(&pVal->value, value->valuestring, pVal->valueLen); + return smlJsonCreateSring(&pVal->value, value->valuestring, pVal->length); } static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { @@ -1754,7 +1778,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { * user configured parameter tsDefaultJSONStrType */ - char *tsDefaultJSONStrType = "binary"; //todo + char *tsDefaultJSONStrType = "nchar"; //todo smlConvertJSONString(kv, tsDefaultJSONStrType, root); break; } @@ -1985,8 +2009,8 @@ static int32_t smlParseInfluxLine(SSmlHandle* info, const char* sql) { (*oneTable)->sTableName = elements.measure; (*oneTable)->sTableNameLen = elements.measureLen; - RandTableName rName = {.tags=(*oneTable)->tags, .sTableName=(*oneTable)->sTableName, .sTableNameLen=(uint8_t)(*oneTable)->sTableNameLen, - .childTableName=(*oneTable)->childTableName}; + RandTableName rName = { (*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, + (*oneTable)->childTableName, 0 }; buildChildTableName(&rName); (*oneTable)->uid = rName.uid; @@ -1994,14 +2018,18 @@ static int32_t smlParseInfluxLine(SSmlHandle* info, const char* sql) { SSmlSTableMeta** tableMeta = (SSmlSTableMeta**)taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta - ret = smlUpdateMeta(*tableMeta, hasTable ? NULL : (*oneTable)->tags, cols, &info->msgBuf); + ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); + if(!hasTable && ret){ + ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); + } if(!ret){ uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id); return TSDB_CODE_SML_INVALID_DATA; } }else{ SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta, (*oneTable)->tags, cols); + smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); + smlInsertMeta(meta->colHash, meta->cols, cols); taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); } @@ -2026,15 +2054,15 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) { } if(info->protocol == TSDB_SML_TELNET_PROTOCOL){ - smlParseTelnetString(info, (const char*)data, tinfo, cols); + ret = smlParseTelnetString(info, (const char*)data, tinfo, cols); }else if(info->protocol == TSDB_SML_JSON_PROTOCOL){ - smlParseJSONString(info, (cJSON *)data, tinfo, cols); + ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); }else{ ASSERT(0); } if(ret != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" smlParseTelnetLine failed", info->id); - smlDestroyTableInfo(tinfo, true); + smlDestroyTableInfo(info, tinfo); taosArrayDestroy(cols); return ret; } @@ -2045,8 +2073,8 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) { } taosHashClear(info->dumplicateKey); - RandTableName rName = {.tags=tinfo->tags, .sTableName=tinfo->sTableName, .sTableNameLen=(uint8_t)tinfo->sTableNameLen, - .childTableName=tinfo->childTableName}; + RandTableName rName = { tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, + tinfo->childTableName, 0 }; buildChildTableName(&rName); tinfo->uid = rName.uid; @@ -2057,20 +2085,24 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) { oneTable = &tinfo; hasTable = false; }else{ - smlDestroyTableInfo(tinfo, true); + smlDestroyTableInfo(info, tinfo); } taosArrayPush((*oneTable)->cols, &cols); SSmlSTableMeta** tableMeta = (SSmlSTableMeta** )taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen); if(tableMeta){ // update meta - ret = smlUpdateMeta(*tableMeta, hasTable ? NULL : (*oneTable)->tags, cols, &info->msgBuf); + ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); + if(!hasTable && ret){ + ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); + } if(!ret){ uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id); return TSDB_CODE_SML_INVALID_DATA; } }else{ SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta, (*oneTable)->tags, cols); + smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); + smlInsertMeta(meta->colHash, meta->cols, cols); taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES); } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 29f965fd7459bc6e1e29e70afd7ee19c7c8d23fd..1e9cb7b24fa89d348d1b615960afa63277902a7a 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -279,7 +279,6 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { } pStmt->exec.autoCreateTbl = false; - pStmt->exec.emptyRes = false; if (keepTable) { return TSDB_CODE_SUCCESS; @@ -298,7 +297,6 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { taosMemoryFree(pStmt->sql.queryRes.userFields); taosMemoryFree(pStmt->sql.sqlStr); qDestroyQuery(pStmt->sql.pQuery); - qDestroyQueryPlan(pStmt->sql.pQueryPlan); taosArrayDestroy(pStmt->sql.nodeList); void* pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); @@ -599,6 +597,8 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fiel int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bInfo.needParse = false; @@ -617,21 +617,42 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STMT_ERR_RET(stmtParseSql(pStmt)); } - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); - if (STMT_TYPE_QUERY == pStmt->sql.type) { - if (NULL == pStmt->sql.pQueryPlan) { - STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); - pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; - pStmt->exec.pRequest->body.pDag = NULL; - STMT_ERR_RET(stmtBackupQueryFields(pStmt)); - } else { - STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); + STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx, pStmt->exec.pRequest->requestId)); + + SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId, + .acctId = pStmt->taos->acctId, + .db = pStmt->exec.pRequest->pDb, + .topicQuery = false, + .pSql = pStmt->sql.sqlStr, + .sqlLen = pStmt->sql.sqlLen, + .pMsg = pStmt->exec.pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pTransporter = pStmt->taos->pAppInfo->pTransporter, + .pStmtCb = NULL, + .pUser = pStmt->taos->user}; + ctx.mgmtEpSet = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); + STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &ctx.pCatalog)); + + STMT_ERR_RET(qStmtParseQuerySql(&ctx, pStmt->sql.pQuery)); + + if (pStmt->sql.pQuery->haveResultSet) { + setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, pStmt->sql.pQuery->numOfResCols); + setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision); } + + TSWAP(pStmt->exec.pRequest->dbList, pStmt->sql.pQuery->pDbList); + TSWAP(pStmt->exec.pRequest->tableList, pStmt->sql.pQuery->pTableList); - STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId, &pStmt->exec.emptyRes)); + //if (STMT_TYPE_QUERY == pStmt->sql.queryRes) { + // STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); + //} + + //STMT_ERR_RET(stmtBackupQueryFields(pStmt)); + + return TSDB_CODE_SUCCESS; } - + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); @@ -736,11 +757,7 @@ int stmtExec(TAOS_STMT *stmt) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); if (STMT_TYPE_QUERY == pStmt->sql.type) { - if (pStmt->exec.emptyRes) { - pStmt->exec.pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - } else { - scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList, NULL); - } + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, NULL); } else { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, (autoCreateTbl ? (void**)&pRsp : NULL)); @@ -839,16 +856,7 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { } if (STMT_TYPE_QUERY == pStmt->sql.type) { - if (NULL == pStmt->sql.pQueryPlan) { - STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); - pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; - pStmt->exec.pRequest->body.pDag = NULL; - STMT_ERR_RET(stmtBackupQueryFields(pStmt)); - } else { - STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); - } - - *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues); + *nums = taosArrayGetSize(pStmt->sql.pQuery->pPlaceholderValues); } else { STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 639f00ab86e7f87fd77e3755fb875f039501242e..2c3f64c5c1e381ca4fdb48b442253247d05d4213 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -316,7 +316,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { void tmqAssignDelayedHbTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); *pTaskType = TMQ_DELAYED_TASK__HB; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -324,7 +324,7 @@ void tmqAssignDelayedHbTask(void* param, void* tmrId) { void tmqAssignDelayedCommitTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); *pTaskType = TMQ_DELAYED_TASK__COMMIT; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -332,7 +332,7 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { void tmqAssignDelayedReportTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t)); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); *pTaskType = TMQ_DELAYED_TASK__REPORT; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -848,7 +848,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); } - SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper)); + SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); goto CREATE_MSG_FAIL; @@ -987,7 +987,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tmqUpdateEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { - SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper)); + SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM); if (pWrapper == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index f579489e62480c1247873c09b954debe63db67d2..b9870633dbb7cbae63c4f5a74555831d442d6f64 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -228,7 +228,7 @@ TEST(testCase, smlParseCols_tag_Test) { ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->valueLen, 17); + ASSERT_EQ(kv->length, 17); ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); taosMemoryFree(kv); @@ -237,7 +237,7 @@ TEST(testCase, smlParseCols_tag_Test) { ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->valueLen, 7); + ASSERT_EQ(kv->length, 7); ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); taosMemoryFree(kv); @@ -259,7 +259,7 @@ TEST(testCase, smlParseCols_tag_Test) { ASSERT_EQ(strncasecmp(kv->key, TAG, strlen(TAG)), 0); ASSERT_EQ(kv->keyLen, strlen(TAG)); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->valueLen, strlen(TAG)); + ASSERT_EQ(kv->length, strlen(TAG)); ASSERT_EQ(strncasecmp(kv->value, TAG, strlen(TAG)), 0); taosMemoryFree(kv); @@ -499,12 +499,14 @@ TEST(testCase, smlProcess_influx_Test) { TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); ASSERT_NE(res, nullptr); int fieldNum = taos_field_count(res); - ASSERT_EQ(fieldNum, 11); + ASSERT_EQ(fieldNum, 5); int rowNum = taos_affected_rows(res); ASSERT_EQ(rowNum, 2); for (int i = 0; i < rowNum; ++i) { TAOS_ROW rows = taos_fetch_row(res); } + taos_free_result(res); + smlDestroyInfo(info); } // different types @@ -530,245 +532,542 @@ TEST(testCase, smlParseLine_error_Test) { }; int ret = smlProcess(info, (char **)sql, sizeof(sql)/sizeof(sql[0])); ASSERT_NE(ret, 0); + destroyRequest(request); + smlDestroyInfo(info); +} + +TEST(testCase, smlGetTimestampLen_Test) { + uint8_t len = smlGetTimestampLen(0); + ASSERT_EQ(len, 1); + + len = smlGetTimestampLen(1); + ASSERT_EQ(len, 1); + + len = smlGetTimestampLen(10); + ASSERT_EQ(len, 2); + + len = smlGetTimestampLen(390); + ASSERT_EQ(len, 3); + + len = smlGetTimestampLen(-1); + ASSERT_EQ(len, 1); + + len = smlGetTimestampLen(-10); + ASSERT_EQ(len, 2); + + len = smlGetTimestampLen(-390); + ASSERT_EQ(len, 3); +} + +TEST(testCase, smlProcess_telnet_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + + const char *sql[5] = { + "sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01", + "sys.if.bytes.out 1479496102 1.3E3 network=tcp", + "sys.procs.running 1479496100 42 host=web01 ", + " sys.procs.running 1479496200 42 host=web01=4" + }; + int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); + ASSERT_EQ(ret, 0); + + TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); + ASSERT_NE(res, nullptr); + int fieldNum = taos_field_count(res); + ASSERT_EQ(fieldNum, 2); + int rowNum = taos_affected_rows(res); + ASSERT_EQ(rowNum, 1); + for (int i = 0; i < rowNum; ++i) { + TAOS_ROW rows = taos_fetch_row(res); + } + taos_free_result(pRes); + + res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961"); + ASSERT_NE(res, nullptr); + fieldNum = taos_field_count(res); + ASSERT_EQ(fieldNum, 2); + rowNum = taos_affected_rows(res); + ASSERT_EQ(rowNum, 2); + for (int i = 0; i < rowNum; ++i) { + TAOS_ROW rows = taos_fetch_row(res); + } + taos_free_result(pRes); + smlDestroyInfo(info); +} + +TEST(testCase, smlProcess_json1_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + + const char *sql = + "[\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 1346846400,\n" + " \"value\": 18,\n" + " \"tags\": {\n" + " \"host\": \"web01\",\n" + " \"dc\": \"lga\"\n" + " }\n" + " },\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 1346846400,\n" + " \"value\": 9,\n" + " \"tags\": {\n" + " \"host\": \"web02\",\n" + " \"dc\": \"lga\"\n" + " }\n" + " }\n" + "]"; + int ret = smlProcess(info, (char **)(&sql), -1); + ASSERT_EQ(ret, 0); + + TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); + ASSERT_NE(res, nullptr); + int fieldNum = taos_field_count(res); + ASSERT_EQ(fieldNum, 2); + // int rowNum = taos_affected_rows(res); + // ASSERT_EQ(rowNum, 1); + // for (int i = 0; i < rowNum; ++i) { + // TAOS_ROW rows = taos_fetch_row(res); + // } + taos_free_result(pRes); + smlDestroyInfo(info); +} + +TEST(testCase, smlProcess_json2_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + const char *sql = + "{\n" + " \"metric\": \"meter_current0\",\n" + " \"timestamp\": {\n" + " \"value\" : 1346846400,\n" + " \"type\" : \"s\"\n" + " },\n" + " \"value\": {\n" + " \"value\" : 10.3,\n" + " \"type\" : \"i64\"\n" + " },\n" + " \"tags\": {\n" + " \"groupid\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"bigint\"\n" + " },\n" + " \"location\": { \n" + " \"value\" : \"北京\",\n" + " \"type\" : \"binary\"\n" + " },\n" + " \"id\": \"d1001\"\n" + " }\n" + "}"; + int32_t ret = smlProcess(info, (char **)(&sql), -1); + ASSERT_EQ(ret, 0); + taos_free_result(pRes); + smlDestroyInfo(info); } - TEST(testCase, smlGetTimestampLen_Test) { - uint8_t len = smlGetTimestampLen(0); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(390); - ASSERT_EQ(len, 3); - - len = smlGetTimestampLen(-1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(-10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(-390); - ASSERT_EQ(len, 3); - } - - TEST(testCase, smlProcess_telnet_Test) { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(taos, nullptr); - - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); - ASSERT_NE(info, nullptr); - - const char *sql[4] = { - "sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", - "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", - "sys.if.bytes.out 1479496102 1.3E3 network=tcp", - "sys.procs.running 1479496100 42 host=web01" - }; - int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_EQ(ret, 0); - - TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); - ASSERT_NE(res, nullptr); - int fieldNum = taos_field_count(res); - ASSERT_EQ(fieldNum, 2); - int rowNum = taos_affected_rows(res); - ASSERT_EQ(rowNum, 1); - for (int i = 0; i < rowNum; ++i) { - TAOS_ROW rows = taos_fetch_row(res); - } - - res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961"); - ASSERT_NE(res, nullptr); - fieldNum = taos_field_count(res); - ASSERT_EQ(fieldNum, 2); - rowNum = taos_affected_rows(res); - ASSERT_EQ(rowNum, 2); - for (int i = 0; i < rowNum; ++i) { - TAOS_ROW rows = taos_fetch_row(res); - } - } - - TEST(testCase, smlProcess_json_Test) { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(taos, nullptr); - - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); - ASSERT_NE(info, nullptr); - - const char *sql = "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 9,\n" - " \"tags\": {\n" - " \"host\": \"web02\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " }\n" - "]"; - int ret = smlProcess(info, (char**)(&sql), -1); - ASSERT_EQ(ret, 0); - - TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); - ASSERT_NE(res, nullptr); - int fieldNum = taos_field_count(res); - ASSERT_EQ(fieldNum, 2); -// int rowNum = taos_affected_rows(res); -// ASSERT_EQ(rowNum, 1); -// for (int i = 0; i < rowNum; ++i) { -// TAOS_ROW rows = taos_fetch_row(res); -// } - - sql = "{\n" - " \"metric\": \"meter_current\",\n" - " \"timestamp\": {\n" - " \"value\" : 1346846400,\n" - " \"type\" : \"s\"\n" - " },\n" - " \"value\": {\n" - " \"value\" : 10.3,\n" - " \"type\" : \"i64\"\n" - " },\n" - " \"tags\": {\n" - " \"groupid\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"bigint\"\n" - " },\n" - " \"location\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"id\": \"d1001\"\n" - " }\n" - "}"; - ret = smlProcess(info, (char**)(&sql), -1); - ASSERT_EQ(ret, 0); - - sql = "{\n" - " \"metric\": \"meter_current\",\n" - " \"timestamp\": {\n" - " \"value\" : 1346846400,\n" - " \"type\" : \"s\"\n" - " },\n" - " \"value\": {\n" - " \"value\" : 10.3,\n" - " \"type\" : \"i64\"\n" - " },\n" - " \"tags\": {\n" - " \"t1\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"bigint\"\n" - " },\n" - " \"t2\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"int\"\n" - " },\n" - " \"t3\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"i16\"\n" - " },\n" - " \"t4\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"i8\"\n" - " },\n" - " \"t5\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"f32\"\n" - " },\n" - " \"t6\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"double\"\n" - " },\n" - " \"t7\": { \n" - " \"value\" : \"8323\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"t8\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"t9\": { \n" - " \"value\" : true,\n" - " \"type\" : \"bool\"\n" - " },\n" - " \"id\": \"d1001\"\n" - " }\n" - "}"; - ret = smlProcess(info, (char**)(&sql), -1); - ASSERT_EQ(ret, 0); - - sql = "{\n" - " \"metric\": \"meter_current\",\n" - " \"timestamp\": {\n" - " \"value\" : 1346846400000,\n" - " \"type\" : \"ms\"\n" - " },\n" - " \"value\": \"ni\",\n" - " \"tags\": {\n" - " \"t1\": { \n" - " \"value\" : 20,\n" - " \"type\" : \"i64\"\n" - " },\n" - " \"t2\": { \n" - " \"value\" : 25,\n" - " \"type\" : \"i32\"\n" - " },\n" - " \"t3\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"smallint\"\n" - " },\n" - " \"t4\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"tinyint\"\n" - " },\n" - " \"t5\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"float\"\n" - " },\n" - " \"t6\": { \n" - " \"value\" : 0.2,\n" - " \"type\" : \"f64\"\n" - " },\n" - " \"t7\": \"nsj\",\n" - " \"t8\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"t9\": false,\n" - " \"id\": \"d1001\"\n" - " }\n" - "}"; - ret = smlProcess(info, (char**)(&sql), -1); - ASSERT_EQ(ret, 0); - } +TEST(testCase, smlProcess_json3_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + const char *sql = + "{\n" + " \"metric\": \"meter_current1\",\n" + " \"timestamp\": {\n" + " \"value\" : 1346846400,\n" + " \"type\" : \"s\"\n" + " },\n" + " \"value\": {\n" + " \"value\" : 10.3,\n" + " \"type\" : \"i64\"\n" + " },\n" + " \"tags\": {\n" + " \"t1\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"bigint\"\n" + " },\n" + " \"t2\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"int\"\n" + " },\n" + " \"t3\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"i16\"\n" + " },\n" + " \"t4\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"i8\"\n" + " },\n" + " \"t5\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"f32\"\n" + " },\n" + " \"t6\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"double\"\n" + " },\n" + " \"t7\": { \n" + " \"value\" : \"8323\",\n" + " \"type\" : \"binary\"\n" + " },\n" + " \"t8\": { \n" + " \"value\" : \"北京\",\n" + " \"type\" : \"nchar\"\n" + " },\n" + " \"t9\": { \n" + " \"value\" : true,\n" + " \"type\" : \"bool\"\n" + " },\n" + " \"id\": \"d1001\"\n" + " }\n" + "}"; + int32_t ret = smlProcess(info, (char **)(&sql), -1); + ASSERT_EQ(ret, 0); + taos_free_result(pRes); + smlDestroyInfo(info); +} + +TEST(testCase, smlProcess_json4_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + const char *sql = "{\n" + " \"metric\": \"meter_current2\",\n" + " \"timestamp\": {\n" + " \"value\" : 1346846500000,\n" + " \"type\" : \"ms\"\n" + " },\n" + " \"value\": \"ni\",\n" + " \"tags\": {\n" + " \"t1\": { \n" + " \"value\" : 20,\n" + " \"type\" : \"i64\"\n" + " },\n" + " \"t2\": { \n" + " \"value\" : 25,\n" + " \"type\" : \"i32\"\n" + " },\n" + " \"t3\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"smallint\"\n" + " },\n" + " \"t4\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"tinyint\"\n" + " },\n" + " \"t5\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"float\"\n" + " },\n" + " \"t6\": { \n" + " \"value\" : 0.2,\n" + " \"type\" : \"f64\"\n" + " },\n" + " \"t7\": \"nsj\",\n" + " \"t8\": { \n" + " \"value\" : \"北京\",\n" + " \"type\" : \"nchar\"\n" + " },\n" + " \"t9\": false,\n" + " \"id\": \"d1001\"\n" + " }\n" + "}"; + int32_t ret = smlProcess(info, (char**)(&sql), -1); + ASSERT_EQ(ret, 0); + taos_free_result(pRes); + smlDestroyInfo(info); +} + +TEST(testCase, smlParseTelnetLine_error_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + + int32_t ret = 0; + const char *sql[19] = { + "sys.procs.running 14794961040 42 host=web01", + "sys.procs.running 14791040 42 host=web01", + "sys.procs.running erere 42 host=web01", + "sys.procs.running 1.6e10 42 host=web01", + "sys.procs.running 1.47949610 42 host=web01", + "sys.procs.running 147949610i 42 host=web01", + "sys.procs.running -147949610 42 host=web01", + "", + " ", + "sys ", + "sys.procs.running 1479496100 42 ", + "sys.procs.running 1479496100 42 host= ", + "sys.procs.running 1479496100 42or host=web01", + "sys.procs.running 1479496100 true host=web01", + "sys.procs.running 1479496100 \"binary\" host=web01", + "sys.procs.running 1479496100 L\"rfr\" host=web01", + "sys.procs.running 1479496100 42 host=web01 cpu= ", + "sys.procs.running 1479496100 42 host=web01 host=w2", + "sys.procs.running 1479496100 42 host=web01 host", + }; + for(int i = 0; i < sizeof(sql)/sizeof(sql[0]); i++){ + ret = smlParseTelnetLine(info, (void*)sql[i]); + ASSERT_NE(ret, 0); + } + + destroyRequest(request); + smlDestroyInfo(info); +} + +TEST(testCase, smlParseTelnetLine_diff_type_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + + const char *sql[2] = { + "sys.procs.running 1479496104000 42 host=web01", + "sys.procs.running 1479496104000 42u8 host=web01" + }; + int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); + ASSERT_NE(ret, 0); + + destroyRequest(request); + smlDestroyInfo(info); +} + +TEST(testCase, smlParseTelnetLine_json_error_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + + int32_t ret = 0; + const char *sql[] = { + "[\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 13468464009999333322222223,\n" + " \"value\": 18,\n" + " \"tags\": {\n" + " \"host\": \"web01\",\n" + " \"dc\": \"lga\"\n" + " }\n" + " },\n" + "]", + "[\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 1346846400i,\n" + " \"value\": 18,\n" + " \"tags\": {\n" + " \"host\": \"web01\",\n" + " \"dc\": \"lga\"\n" + " }\n" + " },\n" + "]", + "[\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 1346846400,\n" + " \"value\": 18,\n" + " \"tags\": {\n" + " \"groupid\": { \n" + " \"value\" : 2,\n" + " \"type\" : \"nchar\"\n" + " },\n" + " \"location\": { \n" + " \"value\" : \"北京\",\n" + " \"type\" : \"binary\"\n" + " },\n" + " \"id\": \"d1001\"\n" + " }\n" + " },\n" + "]", + }; + for(int i = 0; i < sizeof(sql)/sizeof(sql[0]); i++){ + ret = smlParseTelnetLine(info, (void*)sql[i]); + ASSERT_NE(ret, 0); + } + + destroyRequest(request); + smlDestroyInfo(info); +} + +TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + + const char *sql[2] = { + "[\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 1346846400,\n" + " \"value\": 18,\n" + " \"tags\": {\n" + " \"host\": \"lga\"\n" + " }\n" + " },\n" + "]", + "[\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 1346846400,\n" + " \"value\": 18,\n" + " \"tags\": {\n" + " \"host\": 8\n" + " }\n" + " },\n" + "]", + }; + int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); + ASSERT_NE(ret, 0); + + destroyRequest(request); + smlDestroyInfo(info); +} + +TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, nullptr); + + const char *sql[2] = { + "[\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 1346846400,\n" + " \"value\": 18,\n" + " \"tags\": {\n" + " \"host\": \"lga\"\n" + " }\n" + " },\n" + "]", + "[\n" + " {\n" + " \"metric\": \"sys.cpu.nice\",\n" + " \"timestamp\": 1346846400,\n" + " \"value\": \"18\",\n" + " \"tags\": {\n" + " \"host\": \"fff\"\n" + " }\n" + " },\n" + "]", + }; + int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); + ASSERT_NE(ret, 0); + + destroyRequest(request); + smlDestroyInfo(info); +} diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 81682bb734252d85ff8af19eda3edb506d17f8a2..e4e5abe148130b031036ed2101b7d8075ecc687e 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -26,7 +26,7 @@ static const SSysDbTableSchema dnodesSchema[] = { {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, - {.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "support_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0999cb4d2c4be55ac5b151b28b8037b74d585736..025ad0ca0d44597a32868bb46fe99b61fbea255f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -33,18 +33,18 @@ int32_t tsStatusInterval = 1; // second // common int32_t tsMaxShellConns = 50000; -int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second bool tsEnableSlaveQuery = true; bool tsPrintAuth = false; // multi process bool tsMultiProcess = false; -int32_t tsMnodeShmSize = TSDB_MAX_WAL_SIZE * 2; -int32_t tsVnodeShmSize = TSDB_MAX_WAL_SIZE * 10; -int32_t tsQnodeShmSize = TSDB_MAX_WAL_SIZE * 4; -int32_t tsSnodeShmSize = TSDB_MAX_WAL_SIZE * 4; -int32_t tsBnodeShmSize = TSDB_MAX_WAL_SIZE * 4; +int32_t tsMnodeShmSize = TSDB_MAX_WAL_SIZE * 2 + 128; +int32_t tsVnodeShmSize = TSDB_MAX_WAL_SIZE * 10 + 128; +int32_t tsQnodeShmSize = TSDB_MAX_WAL_SIZE * 4 + 128; +int32_t tsSnodeShmSize = TSDB_MAX_WAL_SIZE * 4 + 128; +int32_t tsBnodeShmSize = TSDB_MAX_WAL_SIZE * 4 + 128; +int32_t tsNumOfShmThreads = 1; // queue & threads int32_t tsNumOfRpcThreads = 1; @@ -280,6 +280,7 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1; @@ -351,15 +352,14 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { } static int32_t taosAddServerCfg(SConfig *pCfg) { - if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 4096, 0) != 0) return -1; if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 4096, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxStreamCompDelay", tsMaxStreamComputDelay, 10, 1000000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxFirstStreamCompDelay", tsStreamCompStartDelay, 1000, 1000000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "retryStreamCompDelay", tsRetryStreamCompDelay, 10, 1000000000, 0) != 0) return -1; @@ -371,11 +371,12 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1; if (cfgAddBool(pCfg, "multiProcess", tsMultiProcess, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "vnodeShmSize", tsVnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "qnodeShmSize", tsQnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "snodeShmSize", tsSnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "bnodeShmSize", tsBnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, TSDB_MAX_WAL_SIZE + 128, INT32_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "vnodeShmSize", tsVnodeShmSize, TSDB_MAX_WAL_SIZE + 128, INT32_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "qnodeShmSize", tsQnodeShmSize, TSDB_MAX_WAL_SIZE + 128, INT32_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "snodeShmSize", tsSnodeShmSize, TSDB_MAX_WAL_SIZE + 128, INT32_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "bnodeShmSize", tsBnodeShmSize, TSDB_MAX_WAL_SIZE + 128, INT32_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "mumOfShmThreads", tsNumOfShmThreads, 1, 1024, 0) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, 4); @@ -429,6 +430,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4); if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 1, 1024, 0) != 0) return -1; + tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; + tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_WAL_SIZE * 10L, TSDB_MAX_WAL_SIZE * 10000L); + if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, 1, INT64_MAX, 0) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; @@ -458,6 +463,7 @@ static void taosSetClientLogCfg(SConfig *pCfg) { tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; @@ -533,12 +539,11 @@ static void taosSetSystemCfg(SConfig *pCfg) { static int32_t taosSetServerCfg(SConfig *pCfg) { tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; - tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; - tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32; tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; + tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32; tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32; tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32; @@ -569,6 +574,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; + tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index 42612cecb9531a7d29e92963f7fa7bf582982986..78b70c92882331b697a5dd09abcd1a6991771dd8 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -19,12 +19,16 @@ static SMsgCb tsDefaultMsgCb; -void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } +void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { + // if (tsDefaultMsgCb.pWrapper == NULL) { + tsDefaultMsgCb = *pMsgCb; + //} +} int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { PutToQueueFp fp = pMsgCb->queueFps[qtype]; if (fp != NULL) { - return (*fp)(pMsgCb->pWrapper, pReq); + return (*fp)(pMsgCb->pMgmt, pReq); } else { terrno = TSDB_CODE_INVALID_PTR; return -1; @@ -34,7 +38,7 @@ int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { GetQueueSizeFp fp = pMsgCb->qsizeFp; if (fp != NULL) { - return (*fp)(pMsgCb->pWrapper, vgId, qtype); + return (*fp)(pMsgCb->pMgmt, vgId, qtype); } else { terrno = TSDB_CODE_INVALID_PTR; return -1; @@ -51,7 +55,7 @@ int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) { } } -void tmsgSendRsp(const SRpcMsg* pRsp) { +void tmsgSendRsp(SRpcMsg* pRsp) { SendRspFp fp = tsDefaultMsgCb.sendRspFp; if (fp != NULL) { return (*fp)(tsDefaultMsgCb.pWrapper, pRsp); @@ -60,7 +64,7 @@ void tmsgSendRsp(const SRpcMsg* pRsp) { } } -void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) { +void tmsgSendRedirectRsp(SRpcMsg* pRsp, const SEpSet* pNewEpSet) { SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; if (fp != NULL) { (*fp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet); @@ -69,6 +73,15 @@ void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) { } } +void tmsgSendMnodeRecv(SRpcMsg* pReq, SRpcMsg* pRsp) { + SendMnodeRecvFp fp = tsDefaultMsgCb.sendMnodeRecvFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, pReq, pRsp); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } +} + void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) { RegisterBrokenLinkArgFp fp = pMsgCb->registerBrokenLinkArgFp; if (fp != NULL) { diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 56fbfed8fff294ac482f5ffe25a00192f4bb9880..fa9b6e1e63628593aca1fb27ead10ec6a7bff7d1 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -317,7 +317,11 @@ void buildChildTableName(RandTableName* rName) { for (int j = 0; j < size; ++j) { SSmlKv* tagKv = taosArrayGetP(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); - taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen); + if(IS_VAR_DATA_TYPE(tagKv->type)){ + taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->length); + }else{ + taosStringBuilderAppendStringLen(&sb, (char*)(&(tagKv->value)), tagKv->length); + } } size_t len = 0; char* keyJoined = taosStringBuilderGetResult(&sb, &len); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 3fd2ef8e730d64327f4a75448743f07248e49de3..b3999a49e7462e27582907b4f12aef1b24e2b35c 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -402,7 +402,8 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, - {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, + getStatics_nchr}, }; char tTokenTypeSwitcher[13] = { diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 49ea54e92835f1173e80fef46f6184f258ebdb5e..581686ba904001b80638efd9ab225c3f68e91e09 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,16 +1,17 @@ -add_subdirectory(interface) -add_subdirectory(implement) +add_subdirectory(node_mgmt) +add_subdirectory(node_util) add_subdirectory(mgmt_bnode) add_subdirectory(mgmt_mnode) add_subdirectory(mgmt_qnode) add_subdirectory(mgmt_snode) add_subdirectory(mgmt_vnode) +add_subdirectory(mgmt_dnode) add_subdirectory(test) aux_source_directory(exe EXEC_SRC) add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/implement/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" ) target_link_libraries(taosd dnode) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 4a2d02d25d08b8b1f141e688ac1ee61015db3c73..81df34ef4a827ae16b64d5d422e129a0c42df165 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "dmImp.h" +#include "dmMgmt.h" #include "tconfig.h" #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." @@ -163,14 +163,14 @@ static SDnodeOpt dmGetOpt() { static int32_t dmInitLog() { char logName[12] = {0}; - snprintf(logName, sizeof(logName), "%slog", dmLogName(global.ntype)); + snprintf(logName, sizeof(logName), "%slog", dmNodeLogName(global.ntype)); return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void dmSetProcInfo(int32_t argc, char **argv) { taosSetProcPath(argc, argv); if (global.ntype != DNODE && global.ntype != NODE_END) { - const char *name = dmProcName(global.ntype); + const char *name = dmNodeProcName(global.ntype); taosSetProcName(argc, argv, name); } } diff --git a/source/dnode/mgmt/implement/inc/dmImp.h b/source/dnode/mgmt/implement/inc/dmImp.h deleted file mode 100644 index 8a1a116ab3ab2693921e0fe6a61132bb9b44bb29..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/implement/inc/dmImp.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_IMP_H_ -#define _TD_DND_IMP_H_ - -#include "dmInt.h" - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t dmOpenNode(SMgmtWrapper *pWrapper); -void dmCloseNode(SMgmtWrapper *pWrapper); - -// dmTransport.c -int32_t dmInitServer(SDnode *pDnode); -void dmCleanupServer(SDnode *pDnode); -int32_t dmInitClient(SDnode *pDnode); -void dmCleanupClient(SDnode *pDnode); -SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper); -SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper); -int32_t dmInitMsgHandle(SDnode *pDnode); -void dmSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -void dmSendToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp); - -// dmEps.c -int32_t dmReadEps(SDnode *pDnode); -int32_t dmWriteEps(SDnode *pDnode); -void dmUpdateEps(SDnode *pDnode, SArray *pDnodeEps); - -// dmHandle.c -void dmSendStatusReq(SDnode *pDnode); -int32_t dmProcessConfigReq(SDnode *pDnode, SNodeMsg *pMsg); -int32_t dmProcessAuthRsp(SDnode *pDnode, SNodeMsg *pMsg); -int32_t dmProcessGrantRsp(SDnode *pDnode, SNodeMsg *pMsg); -int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg); -int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg); - -// dmMonitor.c -void dmGetVnodeLoads(SDnode *pDnode, SMonVloadInfo *pInfo); -void dmGetMnodeLoads(SDnode *pDnode, SMonMloadInfo *pInfo); -void dmSendMonitorReport(SDnode *pDnode); - -// dmWorker.c -int32_t dmStartStatusThread(SDnode *pDnode); -void dmStopStatusThread(SDnode *pDnode); -int32_t dmStartMonitorThread(SDnode *pDnode); -void dmStopMonitorThread(SDnode *pDnode); -int32_t dmStartWorker(SDnode *pDnode); -void dmStopWorker(SDnode *pDnode); -int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); - -// mgmt nodes -void dmSetMgmtFp(SMgmtWrapper *pWrapper); -void bmSetMgmtFp(SMgmtWrapper *pWrapper); -void qmSetMgmtFp(SMgmtWrapper *pWrapper); -void smSetMgmtFp(SMgmtWrapper *pWrapper); -void vmSetMgmtFp(SMgmtWrapper *pWrapper); -void mmSetMgmtFp(SMgmtWrapper *pWrapper); - -void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo); -void mmGetMnodeLoads(SMgmtWrapper *pWrapper, SMonMloadInfo *pInfo); -void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo); -void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo); -void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo); -void smGetMonitorInfo(SMgmtWrapper *pWrapper, SMonSmInfo *smInfo); -void bmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonBmInfo *bmInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_IMP_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c deleted file mode 100644 index 097d18679b0a3bd99a6e61ad2d2d935b2ac1c3cb..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmImp.h" - -static void dmUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { - if (pDnode->data.dnodeId == 0 || pDnode->data.clusterId == 0) { - dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); - taosWLockLatch(&pDnode->data.latch); - pDnode->data.dnodeId = pCfg->dnodeId; - pDnode->data.clusterId = pCfg->clusterId; - dmWriteEps(pDnode); - taosWUnLockLatch(&pDnode->data.latch); - } -} - -static int32_t dmProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { - if (pRsp->code != TSDB_CODE_SUCCESS) { - if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pDnode->data.dropped && pDnode->data.dnodeId > 0) { - dInfo("dnode:%d, set to dropped since not exist in mnode", pDnode->data.dnodeId); - pDnode->data.dropped = 1; - dmWriteEps(pDnode); - } - } else { - SStatusRsp statusRsp = {0}; - if (pRsp->pCont != NULL && pRsp->contLen > 0 && - tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { - pDnode->data.dnodeVer = statusRsp.dnodeVer; - dmUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg); - dmUpdateEps(pDnode, statusRsp.pDnodeEps); - } - rpcFreeCont(pRsp->pCont); - tFreeSStatusRsp(&statusRsp); - } - - return TSDB_CODE_SUCCESS; -} - -void dmSendStatusReq(SDnode *pDnode) { - SStatusReq req = {0}; - - taosRLockLatch(&pDnode->data.latch); - req.sver = tsVersion; - req.dnodeVer = pDnode->data.dnodeVer; - req.dnodeId = pDnode->data.dnodeId; - req.clusterId = pDnode->data.clusterId; - if (req.clusterId == 0) req.dnodeId = 0; - req.rebootTime = pDnode->data.rebootTime; - req.updateTime = pDnode->data.updateTime; - req.numOfCores = tsNumOfCores; - req.numOfSupportVnodes = pDnode->data.supportVnodes; - tstrncpy(req.dnodeEp, pDnode->data.localEp, TSDB_EP_LEN); - - req.clusterCfg.statusInterval = tsStatusInterval; - req.clusterCfg.checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); - memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); - taosRUnLockLatch(&pDnode->data.latch); - - SMonVloadInfo vinfo = {0}; - dmGetVnodeLoads(pDnode, &vinfo); - req.pVloads = vinfo.pVloads; - pDnode->data.unsyncedVgId = 0; - pDnode->data.vndState = TAOS_SYNC_STATE_LEADER; - for (int32_t i = 0; i < taosArrayGetSize(req.pVloads); ++i) { - SVnodeLoad *pLoad = taosArrayGet(req.pVloads, i); - if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) { - pDnode->data.unsyncedVgId = pLoad->vgId; - pDnode->data.vndState = pLoad->syncState; - } - } - - SMonMloadInfo minfo = {0}; - dmGetMnodeLoads(pDnode, &minfo); - pDnode->data.isMnode = minfo.isMnode; - pDnode->data.mndState = minfo.load.syncState; - - int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); - tSerializeSStatusReq(pHead, contLen, &req); - tFreeSStatusReq(&req); - - SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)0x9527}; - SRpcMsg rpcRsp = {0}; - - dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); - dmSendToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); - dmProcessStatusRsp(pDnode, &rpcRsp); -} - -int32_t dmProcessAuthRsp(SDnode *pDnode, SNodeMsg *pMsg) { - SRpcMsg *pRsp = &pMsg->rpcMsg; - dError("auth rsp is received, but not supported yet"); - return 0; -} - -int32_t dmProcessGrantRsp(SDnode *pDnode, SNodeMsg *pMsg) { - SRpcMsg *pRsp = &pMsg->rpcMsg; - dError("grant rsp is received, but not supported yet"); - return 0; -} - -int32_t dmProcessConfigReq(SDnode *pDnode, SNodeMsg *pMsg) { - SRpcMsg *pReq = &pMsg->rpcMsg; - SDCfgDnodeReq *pCfg = pReq->pCont; - dError("config req is received, but not supported yet"); - return TSDB_CODE_OPS_NOT_SUPPORT; -} - -int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); - if (pWrapper != NULL) { - dmReleaseWrapper(pWrapper); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; - dError("failed to create node since %s", terrstr()); - return -1; - } - - taosThreadMutexLock(&pDnode->mutex); - pWrapper = &pDnode->wrappers[ntype]; - - if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); - return -1; - } - - int32_t code = (*pWrapper->fp.createFp)(pWrapper, pMsg); - if (code != 0) { - dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); - } else { - dDebug("node:%s, has been created", pWrapper->name); - (void)dmOpenNode(pWrapper); - pWrapper->required = true; - pWrapper->deployed = true; - pWrapper->procType = pDnode->ptype; - } - - taosThreadMutexUnlock(&pDnode->mutex); - return code; -} - -int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); - if (pWrapper == NULL) { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; - dError("failed to drop node since %s", terrstr()); - return -1; - } - - taosThreadMutexLock(&pDnode->mutex); - - int32_t code = (*pWrapper->fp.dropFp)(pWrapper, pMsg); - if (code != 0) { - dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); - } else { - dDebug("node:%s, has been dropped", pWrapper->name); - pWrapper->required = false; - pWrapper->deployed = false; - } - - dmReleaseWrapper(pWrapper); - - if (code == 0) { - dmCloseNode(pWrapper); - taosRemoveDir(pWrapper->path); - } - taosThreadMutexUnlock(&pDnode->mutex); - return code; -} - -static void dmSetMgmtMsgHandle(SMgmtWrapper *pWrapper) { - // Requests handled by DNODE - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, DEFAULT_HANDLE); - - // Requests handled by MNODE - dmSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE); -} - -static int32_t dmStartMgmt(SMgmtWrapper *pWrapper) { - if (dmStartStatusThread(pWrapper->pDnode) != 0) { - return -1; - } - if (dmStartMonitorThread(pWrapper->pDnode) != 0) { - return -1; - } - return 0; -} - -static void dmStopMgmt(SMgmtWrapper *pWrapper) { - dmStopMonitorThread(pWrapper->pDnode); - dmStopStatusThread(pWrapper->pDnode); -} - -static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { - dInfo("dnode-mgmt start to init"); - SDnode *pDnode = pWrapper->pDnode; - - pDnode->data.dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - if (pDnode->data.dnodeHash == NULL) { - dError("failed to init dnode hash"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (dmReadEps(pDnode) != 0) { - dError("failed to read file since %s", terrstr()); - return -1; - } - - if (pDnode->data.dropped) { - dError("dnode will not start since its already dropped"); - return -1; - } - - if (dmStartWorker(pDnode) != 0) { - return -1; - } - - if (dmInitServer(pDnode) != 0) { - dError("failed to init transport since %s", terrstr()); - return -1; - } - dmReportStartup(pDnode, "dnode-transport", "initialized"); - - if (udfStartUdfd(pDnode->data.dnodeId) != 0) { - dError("failed to start udfd"); - } - - dInfo("dnode-mgmt is initialized"); - return 0; -} - -static void dmCleanupMgmt(SMgmtWrapper *pWrapper) { - dInfo("dnode-mgmt start to clean up"); - SDnode *pDnode = pWrapper->pDnode; - - udfStopUdfd(); - - dmStopWorker(pDnode); - - taosWLockLatch(&pDnode->data.latch); - if (pDnode->data.dnodeEps != NULL) { - taosArrayDestroy(pDnode->data.dnodeEps); - pDnode->data.dnodeEps = NULL; - } - if (pDnode->data.dnodeHash != NULL) { - taosHashCleanup(pDnode->data.dnodeHash); - pDnode->data.dnodeHash = NULL; - } - taosWUnLockLatch(&pDnode->data.latch); - - dmCleanupClient(pDnode); - dmCleanupServer(pDnode); - dInfo("dnode-mgmt is cleaned up"); -} - -static int32_t dmRequireMgmt(SMgmtWrapper *pWrapper, bool *required) { - *required = true; - return 0; -} - -void dmSetMgmtFp(SMgmtWrapper *pWrapper) { - SMgmtFp mgmtFp = {0}; - mgmtFp.openFp = dmInitMgmt; - mgmtFp.closeFp = dmCleanupMgmt; - mgmtFp.startFp = dmStartMgmt; - mgmtFp.stopFp = dmStopMgmt; - mgmtFp.requiredFp = dmRequireMgmt; - - dmSetMgmtMsgHandle(pWrapper); - pWrapper->name = "dnode"; - pWrapper->fp = mgmtFp; -} diff --git a/source/dnode/mgmt/implement/src/dmMonitor.c b/source/dnode/mgmt/implement/src/dmMonitor.c deleted file mode 100644 index 8543310eb54479fe4edd331b8d38b840892cac20..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/implement/src/dmMonitor.c +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmImp.h" - -static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { - pInfo->protocol = 1; - pInfo->dnode_id = pDnode->data.dnodeId; - pInfo->cluster_id = pDnode->data.clusterId; - tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); -} - -static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); - pInfo->has_mnode = pDnode->wrappers[MNODE].required; - pInfo->has_qnode = pDnode->wrappers[QNODE].required; - pInfo->has_snode = pDnode->wrappers[SNODE].required; - pInfo->has_bnode = pDnode->wrappers[BNODE].required; - tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); - pInfo->logdir.size = tsLogSpace.size; - tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); - pInfo->tempdir.size = tsTempSpace.size; -} - -static void dmGetMonitorInfo(SDnode *pDnode, SMonDmInfo *pInfo) { - dmGetMonitorBasicInfo(pDnode, &pInfo->basic); - dmGetMonitorSysInfo(&pInfo->sys); - dmGetMonitorDnodeInfo(pDnode, &pInfo->dnode); -} - -void dmSendMonitorReport(SDnode *pDnode) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); - - SMonDmInfo dmInfo = {0}; - SMonMmInfo mmInfo = {0}; - SMonVmInfo vmInfo = {0}; - SMonQmInfo qmInfo = {0}; - SMonSmInfo smInfo = {0}; - SMonBmInfo bmInfo = {0}; - - SRpcMsg req = {0}; - SRpcMsg rsp; - SEpSet epset = {.inUse = 0, .numOfEps = 1}; - tstrncpy(epset.eps[0].fqdn, pDnode->data.localFqdn, TSDB_FQDN_LEN); - epset.eps[0].port = tsServerPort; - - SMgmtWrapper *pWrapper = NULL; - dmGetMonitorInfo(pDnode, &dmInfo); - - bool getFromAPI = !tsMultiProcess; - pWrapper = &pDnode->wrappers[MNODE]; - if (getFromAPI) { - if (dmMarkWrapper(pWrapper) == 0) { - mmGetMonitorInfo(pWrapper, &mmInfo); - dmReleaseWrapper(pWrapper); - } - } else { - if (pWrapper->required) { - req.msgType = TDMT_MON_MM_INFO; - dmSendRecv(pDnode, &epset, &req, &rsp); - if (rsp.code == 0 && rsp.contLen > 0) { - tDeserializeSMonMmInfo(rsp.pCont, rsp.contLen, &mmInfo); - } - rpcFreeCont(rsp.pCont); - } - } - - pWrapper = &pDnode->wrappers[VNODE]; - if (getFromAPI) { - if (dmMarkWrapper(pWrapper) == 0) { - vmGetMonitorInfo(pWrapper, &vmInfo); - dmReleaseWrapper(pWrapper); - } - } else { - if (pWrapper->required) { - req.msgType = TDMT_MON_VM_INFO; - dmSendRecv(pDnode, &epset, &req, &rsp); - if (rsp.code == 0 && rsp.contLen > 0) { - tDeserializeSMonVmInfo(rsp.pCont, rsp.contLen, &vmInfo); - } - rpcFreeCont(rsp.pCont); - } - } - - pWrapper = &pDnode->wrappers[QNODE]; - if (getFromAPI) { - if (dmMarkWrapper(pWrapper) == 0) { - qmGetMonitorInfo(pWrapper, &qmInfo); - dmReleaseWrapper(pWrapper); - } - } else { - if (pWrapper->required) { - req.msgType = TDMT_MON_QM_INFO; - dmSendRecv(pDnode, &epset, &req, &rsp); - if (rsp.code == 0 && rsp.contLen > 0) { - tDeserializeSMonQmInfo(rsp.pCont, rsp.contLen, &qmInfo); - } - rpcFreeCont(rsp.pCont); - } - } - - pWrapper = &pDnode->wrappers[SNODE]; - if (getFromAPI) { - if (dmMarkWrapper(pWrapper) == 0) { - smGetMonitorInfo(pWrapper, &smInfo); - dmReleaseWrapper(pWrapper); - } - } else { - if (pWrapper->required) { - req.msgType = TDMT_MON_SM_INFO; - dmSendRecv(pDnode, &epset, &req, &rsp); - if (rsp.code == 0 && rsp.contLen > 0) { - tDeserializeSMonSmInfo(rsp.pCont, rsp.contLen, &smInfo); - } - rpcFreeCont(rsp.pCont); - } - } - - pWrapper = &pDnode->wrappers[BNODE]; - if (getFromAPI) { - if (dmMarkWrapper(pWrapper) == 0) { - bmGetMonitorInfo(pWrapper, &bmInfo); - dmReleaseWrapper(pWrapper); - } - } else { - if (pWrapper->required) { - req.msgType = TDMT_MON_BM_INFO; - dmSendRecv(pDnode, &epset, &req, &rsp); - if (rsp.code == 0 && rsp.contLen > 0) { - tDeserializeSMonBmInfo(rsp.pCont, rsp.contLen, &bmInfo); - } - rpcFreeCont(rsp.pCont); - } - } - - monSetDmInfo(&dmInfo); - monSetMmInfo(&mmInfo); - monSetVmInfo(&vmInfo); - monSetQmInfo(&qmInfo); - monSetSmInfo(&smInfo); - monSetBmInfo(&bmInfo); - tFreeSMonMmInfo(&mmInfo); - tFreeSMonVmInfo(&vmInfo); - tFreeSMonQmInfo(&qmInfo); - tFreeSMonSmInfo(&smInfo); - tFreeSMonBmInfo(&bmInfo); - monSendReport(); -} - -void dmGetVnodeLoads(SDnode *pDnode, SMonVloadInfo *pInfo) { - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper == NULL) return; - - bool getFromAPI = !tsMultiProcess; - if (getFromAPI) { - vmGetVnodeLoads(pWrapper, pInfo); - } else { - SRpcMsg req = {.msgType = TDMT_MON_VM_LOAD}; - SRpcMsg rsp = {0}; - SEpSet epset = {.inUse = 0, .numOfEps = 1}; - tstrncpy(epset.eps[0].fqdn, pDnode->data.localFqdn, TSDB_FQDN_LEN); - epset.eps[0].port = tsServerPort; - - dmSendRecv(pDnode, &epset, &req, &rsp); - if (rsp.code == 0 && rsp.contLen > 0) { - tDeserializeSMonVloadInfo(rsp.pCont, rsp.contLen, pInfo); - } - rpcFreeCont(rsp.pCont); - } - dmReleaseWrapper(pWrapper); -} - -void dmGetMnodeLoads(SDnode *pDnode, SMonMloadInfo *pInfo) { - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); - if (pWrapper == NULL) { - pInfo->isMnode = 0; - return; - } - - bool getFromAPI = !tsMultiProcess; - if (getFromAPI) { - mmGetMnodeLoads(pWrapper, pInfo); - } else { - SRpcMsg req = {.msgType = TDMT_MON_MM_LOAD}; - SRpcMsg rsp = {0}; - SEpSet epset = {.inUse = 0, .numOfEps = 1}; - tstrncpy(epset.eps[0].fqdn, pDnode->data.localFqdn, TSDB_FQDN_LEN); - epset.eps[0].port = tsServerPort; - - dmSendRecv(pDnode, &epset, &req, &rsp); - if (rsp.code == 0 && rsp.contLen > 0) { - tDeserializeSMonMloadInfo(rsp.pCont, rsp.contLen, pInfo); - } - rpcFreeCont(rsp.pCont); - } - dmReleaseWrapper(pWrapper); -} diff --git a/source/dnode/mgmt/implement/src/dmObj.c b/source/dnode/mgmt/implement/src/dmObj.c deleted file mode 100644 index a43439d4658beda0465e5a3c2179d7e71e755395..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/implement/src/dmObj.c +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmImp.h" - -static int32_t dmInitVars(SDnode *pDnode, const SDnodeOpt *pOption) { - pDnode->data.dnodeId = 0; - pDnode->data.clusterId = 0; - pDnode->data.dnodeVer = 0; - pDnode->data.updateTime = 0; - pDnode->data.rebootTime = taosGetTimestampMs(); - pDnode->data.dropped = 0; - pDnode->data.localEp = strdup(pOption->localEp); - pDnode->data.localFqdn = strdup(pOption->localFqdn); - pDnode->data.firstEp = strdup(pOption->firstEp); - pDnode->data.secondEp = strdup(pOption->secondEp); - pDnode->data.dataDir = strdup(pOption->dataDir); - pDnode->data.disks = pOption->disks; - pDnode->data.numOfDisks = pOption->numOfDisks; - pDnode->data.supportVnodes = pOption->numOfSupportVnodes; - pDnode->data.serverPort = pOption->serverPort; - pDnode->ntype = pOption->ntype; - - if (pDnode->data.dataDir == NULL || pDnode->data.localEp == NULL || pDnode->data.localFqdn == NULL || - pDnode->data.firstEp == NULL || pDnode->data.secondEp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_END) { - pDnode->data.lockfile = dmCheckRunning(pDnode->data.dataDir); - if (pDnode->data.lockfile == NULL) { - return -1; - } - } - - taosInitRWLatch(&pDnode->data.latch); - taosThreadMutexInit(&pDnode->mutex, NULL); - return 0; -} - -static void dmClearVars(SDnode *pDnode) { - for (EDndNodeType n = DNODE; n < NODE_END; ++n) { - SMgmtWrapper *pMgmt = &pDnode->wrappers[n]; - taosMemoryFreeClear(pMgmt->path); - } - if (pDnode->data.lockfile != NULL) { - taosUnLockFile(pDnode->data.lockfile); - taosCloseFile(&pDnode->data.lockfile); - pDnode->data.lockfile = NULL; - } - taosMemoryFreeClear(pDnode->data.localEp); - taosMemoryFreeClear(pDnode->data.localFqdn); - taosMemoryFreeClear(pDnode->data.firstEp); - taosMemoryFreeClear(pDnode->data.secondEp); - taosMemoryFreeClear(pDnode->data.dataDir); - taosThreadMutexDestroy(&pDnode->mutex); - memset(&pDnode->mutex, 0, sizeof(pDnode->mutex)); - taosMemoryFree(pDnode); - dDebug("dnode memory is cleared, data:%p", pDnode); -} - -SDnode *dmCreate(const SDnodeOpt *pOption) { - dDebug("start to create dnode"); - int32_t code = -1; - char path[PATH_MAX] = {0}; - SDnode *pDnode = NULL; - - pDnode = taosMemoryCalloc(1, sizeof(SDnode)); - if (pDnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - - if (dmInitVars(pDnode, pOption) != 0) { - dError("failed to init variables since %s", terrstr()); - goto _OVER; - } - - dmSetStatus(pDnode, DND_STAT_INIT); - dmSetMgmtFp(&pDnode->wrappers[DNODE]); - mmSetMgmtFp(&pDnode->wrappers[MNODE]); - vmSetMgmtFp(&pDnode->wrappers[VNODE]); - qmSetMgmtFp(&pDnode->wrappers[QNODE]); - smSetMgmtFp(&pDnode->wrappers[SNODE]); - bmSetMgmtFp(&pDnode->wrappers[BNODE]); - - for (EDndNodeType n = DNODE; n < NODE_END; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - snprintf(path, sizeof(path), "%s%s%s", pDnode->data.dataDir, TD_DIRSEP, pWrapper->name); - pWrapper->path = strdup(path); - pWrapper->procShm.id = -1; - pWrapper->pDnode = pDnode; - pWrapper->ntype = n; - pWrapper->procType = DND_PROC_SINGLE; - taosInitRWLatch(&pWrapper->latch); - - if (pWrapper->path == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - - if (n != DNODE && dmReadShmFile(pWrapper) != 0) { - dError("node:%s, failed to read shm file since %s", pWrapper->name, terrstr()); - goto _OVER; - } - } - - if (dmInitMsgHandle(pDnode) != 0) { - dError("failed to init msg handles since %s", terrstr()); - goto _OVER; - } - - dInfo("dnode is created, data:%p", pDnode); - code = 0; - -_OVER: - if (code != 0 && pDnode) { - dmClearVars(pDnode); - pDnode = NULL; - dError("failed to create dnode since %s", terrstr()); - } - - return pDnode; -} - -void dmClose(SDnode *pDnode) { - if (pDnode == NULL) return; - dmClearVars(pDnode); - dInfo("dnode is closed, data:%p", pDnode); -} diff --git a/source/dnode/mgmt/implement/src/dmWorker.c b/source/dnode/mgmt/implement/src/dmWorker.c deleted file mode 100644 index 72b21115918641a7962619118ff7487f22cad126..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/implement/src/dmWorker.c +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmImp.h" - -static void *dmStatusThreadFp(void *param) { - SDnode *pDnode = param; - int64_t lastTime = taosGetTimestampMs(); - - setThreadName("dnode-status"); - - while (1) { - taosThreadTestCancel(); - taosMsleep(200); - - if (pDnode->status != DND_STAT_RUNNING || pDnode->data.dropped) { - continue; - } - - int64_t curTime = taosGetTimestampMs(); - float interval = (curTime - lastTime) / 1000.0f; - if (interval >= tsStatusInterval) { - dmSendStatusReq(pDnode); - lastTime = curTime; - } - } - - return NULL; -} - -static void *dmMonitorThreadFp(void *param) { - SDnode *pDnode = param; - int64_t lastTime = taosGetTimestampMs(); - - setThreadName("dnode-monitor"); - - while (1) { - taosThreadTestCancel(); - taosMsleep(200); - - if (pDnode->status != DND_STAT_RUNNING || pDnode->data.dropped) { - continue; - } - - int64_t curTime = taosGetTimestampMs(); - float interval = (curTime - lastTime) / 1000.0f; - if (interval >= tsMonitorInterval) { - dmSendMonitorReport(pDnode); - lastTime = curTime; - } - } - - return NULL; -} - -int32_t dmStartStatusThread(SDnode *pDnode) { - pDnode->data.statusThreadId = taosCreateThread(dmStatusThreadFp, pDnode); - if (pDnode->data.statusThreadId == NULL) { - dError("failed to init dnode status thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dmReportStartup(pDnode, "dnode-status", "initialized"); - return 0; -} - -void dmStopStatusThread(SDnode *pDnode) { - if (pDnode->data.statusThreadId != NULL) { - taosDestoryThread(pDnode->data.statusThreadId); - pDnode->data.statusThreadId = NULL; - } -} - -int32_t dmStartMonitorThread(SDnode *pDnode) { - pDnode->data.monitorThreadId = taosCreateThread(dmMonitorThreadFp, pDnode); - if (pDnode->data.monitorThreadId == NULL) { - dError("failed to init dnode monitor thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dmReportStartup(pDnode, "dnode-monitor", "initialized"); - return 0; -} - -void dmStopMonitorThread(SDnode *pDnode) { - if (pDnode->data.monitorThreadId != NULL) { - taosDestoryThread(pDnode->data.monitorThreadId); - pDnode->data.monitorThreadId = NULL; - } -} - -static void dmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { - SDnode *pDnode = pInfo->ahandle; - - int32_t code = -1; - tmsg_t msgType = pMsg->rpcMsg.msgType; - dTrace("msg:%p, will be processed in dnode-mgmt queue", pMsg); - - switch (msgType) { - case TDMT_DND_CONFIG_DNODE: - code = dmProcessConfigReq(pDnode, pMsg); - break; - case TDMT_MND_AUTH_RSP: - code = dmProcessAuthRsp(pDnode, pMsg); - break; - case TDMT_MND_GRANT_RSP: - code = dmProcessGrantRsp(pDnode, pMsg); - break; - case TDMT_DND_CREATE_MNODE: - code = dmProcessCreateNodeReq(pDnode, MNODE, pMsg); - break; - case TDMT_DND_DROP_MNODE: - code = dmProcessDropNodeReq(pDnode, MNODE, pMsg); - break; - case TDMT_DND_CREATE_QNODE: - code = dmProcessCreateNodeReq(pDnode, QNODE, pMsg); - break; - case TDMT_DND_DROP_QNODE: - code = dmProcessDropNodeReq(pDnode, QNODE, pMsg); - break; - case TDMT_DND_CREATE_SNODE: - code = dmProcessCreateNodeReq(pDnode, SNODE, pMsg); - break; - case TDMT_DND_DROP_SNODE: - code = dmProcessDropNodeReq(pDnode, SNODE, pMsg); - break; - case TDMT_DND_CREATE_BNODE: - code = dmProcessCreateNodeReq(pDnode, BNODE, pMsg); - break; - case TDMT_DND_DROP_BNODE: - code = dmProcessDropNodeReq(pDnode, BNODE, pMsg); - break; - default: - break; - } - - if (msgType & 1u) { - if (code != 0 && terrno != 0) code = terrno; - SRpcMsg rsp = { - .handle = pMsg->rpcMsg.handle, - .ahandle = pMsg->rpcMsg.ahandle, - .code = code, - .refId = pMsg->rpcMsg.refId, - }; - rpcSendResponse(&rsp); - } - - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - rpcFreeCont(pMsg->rpcMsg.pCont); - taosFreeQitem(pMsg); -} - -int32_t dmStartWorker(SDnode *pDnode) { - SSingleWorkerCfg cfg = { - .min = 1, - .max = 1, - .name = "dnode-mgmt", - .fp = (FItem)dmProcessMgmtQueue, - .param = pDnode, - }; - if (tSingleWorkerInit(&pDnode->data.mgmtWorker, &cfg) != 0) { - dError("failed to start dnode-mgmt worker since %s", terrstr()); - return -1; - } - - dDebug("dnode workers are initialized"); - return 0; -} - -void dmStopWorker(SDnode *pDnode) { - tSingleWorkerCleanup(&pDnode->data.mgmtWorker); - dDebug("dnode workers are closed"); -} - -int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSingleWorker *pWorker = &pWrapper->pDnode->data.mgmtWorker; - dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); - taosWriteQitem(pWorker->queue, pMsg); - return 0; -} diff --git a/source/dnode/mgmt/interface/CMakeLists.txt b/source/dnode/mgmt/interface/CMakeLists.txt deleted file mode 100644 index a99fc2703d468197149d95fbad23fdcfd5066650..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/interface/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -aux_source_directory(src DNODE_INTERFACE) -add_library(dnode_interface STATIC ${DNODE_INTERFACE}) -target_include_directories( - dnode_interface - PUBLIC "${TD_SOURCE_DIR}/include/dnode/mgmt" - PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - dnode_interface cjson mnode vnode qnode snode bnode wal sync taos_static tfs monitor -) \ No newline at end of file diff --git a/source/dnode/mgmt/interface/inc/dmDef.h b/source/dnode/mgmt/interface/inc/dmDef.h deleted file mode 100644 index 445e1d42f5d6bd96d2fe93aac84d6e24683313f5..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/interface/inc/dmDef.h +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DM_DEF_H_ -#define _TD_DM_DEF_H_ - -#include "uv.h" -#include "dmLog.h" - -#include "cJSON.h" -#include "tcache.h" -#include "tcrc32c.h" -#include "tdatablock.h" -#include "tglobal.h" -#include "thash.h" -#include "tlockfree.h" -#include "tlog.h" -#include "tmsg.h" -#include "tmsgcb.h" -#include "tprocess.h" -#include "tqueue.h" -#include "trpc.h" -#include "tthread.h" -#include "ttime.h" -#include "tworker.h" - -#include "dnode.h" -#include "mnode.h" -#include "monitor.h" -#include "sync.h" - -#include "libs/function/function.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum { DNODE, VNODE, QNODE, SNODE, MNODE, BNODE, NODE_END } EDndNodeType; -typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndRunStatus; -typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EDndEnvStatus; -typedef enum { DND_PROC_SINGLE, DND_PROC_CHILD, DND_PROC_PARENT } EDndProcType; - -typedef int32_t (*NodeMsgFp)(struct SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -typedef int32_t (*OpenNodeFp)(struct SMgmtWrapper *pWrapper); -typedef void (*CloseNodeFp)(struct SMgmtWrapper *pWrapper); -typedef int32_t (*StartNodeFp)(struct SMgmtWrapper *pWrapper); -typedef void (*StopNodeFp)(struct SMgmtWrapper *pWrapper); -typedef int32_t (*CreateNodeFp)(struct SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -typedef int32_t (*DropNodeFp)(struct SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -typedef int32_t (*RequireNodeFp)(struct SMgmtWrapper *pWrapper, bool *required); - -typedef struct { - SMgmtWrapper *pQndWrapper; - SMgmtWrapper *pMndWrapper; - SMgmtWrapper *pNdWrapper; -} SMsgHandle; - -typedef struct { - OpenNodeFp openFp; - CloseNodeFp closeFp; - StartNodeFp startFp; - StopNodeFp stopFp; - CreateNodeFp createFp; - DropNodeFp dropFp; - RequireNodeFp requiredFp; -} SMgmtFp; - -typedef struct SMgmtWrapper { - SDnode *pDnode; - struct { - const char *name; - char *path; - int32_t refCount; - SRWLatch latch; - EDndNodeType ntype; - bool deployed; - bool required; - SMgmtFp fp; - void *pMgmt; - }; - struct { - EDndProcType procType; - int32_t procId; - SProcObj *procObj; - SShm procShm; - }; - struct { - int8_t msgVgIds[TDMT_MAX]; // Handle the case where the same message type is distributed to qnode or vnode - NodeMsgFp msgFps[TDMT_MAX]; - }; -} SMgmtWrapper; - -typedef struct { - void *serverRpc; - void *clientRpc; - SMsgHandle msgHandles[TDMT_MAX]; -} SDnodeTrans; - -typedef struct { - int32_t dnodeId; - int64_t clusterId; - int64_t dnodeVer; - int64_t updateTime; - int64_t rebootTime; - int32_t unsyncedVgId; - ESyncState vndState; - ESyncState mndState; - bool isMnode; - bool dropped; - SEpSet mnodeEps; - SArray *dnodeEps; - SHashObj *dnodeHash; - TdThread *statusThreadId; - TdThread *monitorThreadId; - SRWLatch latch; - SSingleWorker mgmtWorker; - SMsgCb msgCb; - SDnode *pDnode; - TdFilePtr lockfile; - char *localEp; - char *localFqdn; - char *firstEp; - char *secondEp; - char *dataDir; - SDiskCfg *disks; - int32_t numOfDisks; - int32_t supportVnodes; - uint16_t serverPort; -} SDnodeData; - -typedef struct { - char name[TSDB_STEP_NAME_LEN]; - char desc[TSDB_STEP_DESC_LEN]; -} SStartupInfo; - -typedef struct SUdfdData { - bool startCalled; - bool needCleanUp; - uv_loop_t loop; - uv_thread_t thread; - uv_barrier_t barrier; - uv_process_t process; - int spawnErr; - uv_pipe_t ctrlPipe; - uv_async_t stopAsync; - int32_t stopCalled; - - int32_t dnodeId; -} SUdfdData; - -typedef struct SDnode { - EDndProcType ptype; - EDndNodeType ntype; - EDndRunStatus status; - EDndEvent event; - SStartupInfo startup; - SDnodeTrans trans; - SDnodeData data; - SUdfdData udfdData; - TdThreadMutex mutex; - SMgmtWrapper wrappers[NODE_END]; -} SDnode; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DM_DEF_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/interface/inc/dmInt.h b/source/dnode/mgmt/interface/inc/dmInt.h deleted file mode 100644 index b56edd26300f19f39551a7f6911d8c7260c34171..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/interface/inc/dmInt.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DM_INT_H_ -#define _TD_DM_INT_H_ - -#include "dmDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// dmInt.c -SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType nType); -int32_t dmMarkWrapper(SMgmtWrapper *pWrapper); -void dmReleaseWrapper(SMgmtWrapper *pWrapper); -const char *dmStatName(EDndRunStatus stat); -const char *dmLogName(EDndNodeType ntype); -const char *dmProcName(EDndNodeType ntype); -const char *dmEventName(EDndEvent ev); - -void dmSetStatus(SDnode *pDnode, EDndRunStatus stat); -void dmSetEvent(SDnode *pDnode, EDndEvent event); -void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); -void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc); -void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg); -void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg); -void dmGetMonitorSysInfo(SMonSysInfo *pInfo); - -// dmFile.c -int32_t dmReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); -int32_t dmWriteFile(SMgmtWrapper *pWrapper, bool deployed); -TdFilePtr dmCheckRunning(const char *dataDir); -int32_t dmReadShmFile(SMgmtWrapper *pWrapper); -int32_t dmWriteShmFile(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DM_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/interface/inc/dmLog.h b/source/dnode/mgmt/interface/inc/dmLog.h deleted file mode 100644 index c21933fc01fd19624f1b763e4191ce4456411494..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/interface/inc/dmLog.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DM_LOG_H_ -#define _TD_DM_LOG_H_ - -#include "tlog.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} -#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DM_LOG_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/interface/src/dmInt.c b/source/dnode/mgmt/interface/src/dmInt.c deleted file mode 100644 index f8e23ad26212c093ab11b1c83fea085094156ea5..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/interface/src/dmInt.c +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmInt.h" - -const char *dmStatName(EDndRunStatus status) { - switch (status) { - case DND_STAT_INIT: - return "init"; - case DND_STAT_RUNNING: - return "running"; - case DND_STAT_STOPPED: - return "stopped"; - default: - return "UNKNOWN"; - } -} - -const char *dmLogName(EDndNodeType ntype) { - switch (ntype) { - case VNODE: - return "vnode"; - case QNODE: - return "qnode"; - case SNODE: - return "snode"; - case MNODE: - return "mnode"; - case BNODE: - return "bnode"; - default: - return "taosd"; - } -} - -const char *dmProcName(EDndNodeType ntype) { - switch (ntype) { - case VNODE: - return "taosv"; - case QNODE: - return "taosq"; - case SNODE: - return "taoss"; - case MNODE: - return "taosm"; - case BNODE: - return "taosb"; - default: - return "taosd"; - } -} - -const char *dmEventName(EDndEvent ev) { - switch (ev) { - case DND_EVENT_START: - return "start"; - case DND_EVENT_STOP: - return "stop"; - case DND_EVENT_CHILD: - return "child"; - default: - return "UNKNOWN"; - } -} - -void dmSetStatus(SDnode *pDnode, EDndRunStatus status) { - if (pDnode->status != status) { - dDebug("dnode status set from %s to %s", dmStatName(pDnode->status), dmStatName(status)); - pDnode->status = status; - } -} - -void dmSetEvent(SDnode *pDnode, EDndEvent event) { - if (event == DND_EVENT_STOP) { - pDnode->event = event; - } -} - -void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { - pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; - pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; -} - -SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; - SMgmtWrapper *pRetWrapper = pWrapper; - - taosRLockLatch(&pWrapper->latch); - if (pWrapper->deployed) { - int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); - dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount); - } else { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; - pRetWrapper = NULL; - } - taosRUnLockLatch(&pWrapper->latch); - - return pRetWrapper; -} - -int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) { - int32_t code = 0; - - taosRLockLatch(&pWrapper->latch); - if (pWrapper->deployed || (pWrapper->procType == DND_PROC_PARENT && pWrapper->required)) { - int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); - dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount); - } else { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; - code = -1; - } - taosRUnLockLatch(&pWrapper->latch); - - return code; -} - -void dmReleaseWrapper(SMgmtWrapper *pWrapper) { - if (pWrapper == NULL) return; - - taosRLockLatch(&pWrapper->latch); - int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); - taosRUnLockLatch(&pWrapper->latch); - dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount); -} - -void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupInfo *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - dInfo("step:%s, %s", pStartup->name, pStartup->desc); -} - -void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc) { - dmReportStartup(pWrapper->pDnode, pName, pDesc); -} - -static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) { - pStatus->details[0] = 0; - - if (pDnode->status == DND_STAT_INIT) { - pStatus->statusCode = TSDB_SRV_STATUS_NETWORK_OK; - snprintf(pStatus->details, sizeof(pStatus->details), "%s: %s", pDnode->startup.name, pDnode->startup.desc); - } else if (pDnode->status == DND_STAT_STOPPED) { - pStatus->statusCode = TSDB_SRV_STATUS_EXTING; - } else { - SDnodeData *pData = &pDnode->data; - if (pData->isMnode && pData->mndState != TAOS_SYNC_STATE_LEADER && pData->mndState == TAOS_SYNC_STATE_FOLLOWER) { - pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; - snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(pData->mndState)); - } else if (pData->unsyncedVgId != 0 && pData->vndState != TAOS_SYNC_STATE_LEADER && - pData->vndState != TAOS_SYNC_STATE_FOLLOWER) { - pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; - snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pData->unsyncedVgId, - syncStr(pData->vndState)); - } else { - pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK; - } - } -} - -void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("net test req is received"); - SRpcMsg rsp = {.handle = pReq->handle, .refId = pReq->refId, .ahandle = pReq->ahandle, .code = 0}; - rsp.pCont = rpcMallocCont(pReq->contLen); - if (rsp.pCont == NULL) { - rsp.code = TSDB_CODE_OUT_OF_MEMORY; - } else { - rsp.contLen = pReq->contLen; - } - rpcSendResponse(&rsp); - rpcFreeCont(pReq->pCont); -} - -void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("server status req is received"); - - SServerStatusRsp statusRsp = {0}; - dmGetServerStatus(pDnode, &statusRsp); - - SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle, .refId = pReq->refId}; - int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); - if (rspLen < 0) { - rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - - void *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) { - rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - - tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp); - rspMsg.pCont = pRsp; - rspMsg.contLen = rspLen; - -_OVER: - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); -} - -void dmGetMonitorSysInfo(SMonSysInfo *pInfo) { - taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); - taosGetCpuCores(&pInfo->cpu_cores); - taosGetProcMemory(&pInfo->mem_engine); - taosGetSysMemory(&pInfo->mem_system); - pInfo->mem_total = tsTotalMemoryKB; - pInfo->disk_engine = 0; - pInfo->disk_used = tsDataSpace.size.used; - pInfo->disk_total = tsDataSpace.size.total; - taosGetCardInfoDelta(&pInfo->net_in, &pInfo->net_out); - taosGetProcIODelta(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); -} diff --git a/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt index 6a447dccf89b9cca83b2cd3df8309c9a3cec9fe0..0a6cf52fb86dfd7ac24332631f2b9a1ee6c2c572 100644 --- a/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt @@ -5,5 +5,5 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mgmt_bnode dnode_interface + mgmt_bnode node_util ) \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h b/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h index 3adcc1206bf7f8f9e5360102e49b90771344178e..1923d049a581742c5e44e93d7bf137283cdf99e8 100644 --- a/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h +++ b/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_BNODE_INT_H_ #define _TD_DND_BNODE_INT_H_ -#include "dmInt.h" +#include "dmUtil.h" #include "bnode.h" @@ -26,24 +26,25 @@ extern "C" { typedef struct SBnodeMgmt { SBnode *pBnode; - SDnode *pDnode; - SMgmtWrapper *pWrapper; + SMsgCb msgCb; const char *path; + const char *name; + int32_t dnodeId; SMultiWorker writeWorker; SSingleWorker monitorWorker; } SBnodeMgmt; // bmHandle.c -void bmInitMsgHandle(SMgmtWrapper *pWrapper); -int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); +SArray *bmGetMsgHandles(); +int32_t bmProcessCreateReq(const SMgmtInputOpt *pInput, SNodeMsg *pMsg); +int32_t bmProcessDropReq(SBnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t bmProcessGetMonBmInfoReq(SBnodeMgmt *pMgmt, SNodeMsg *pReq); // bmWorker.c int32_t bmStartWorker(SBnodeMgmt *pMgmt); void bmStopWorker(SBnodeMgmt *pMgmt); -int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t bmPutNodeMsgToWriteQueue(SBnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t bmPutNodeMsgToMonitorQueue(SBnodeMgmt *pMgmt, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c b/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c index 3b314b1d2b796664c893ebad40837db40da3b311..d87f832262374ec7d8c49dc5fe08b87cd2b2e0db 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c @@ -16,12 +16,12 @@ #define _DEFAULT_SOURCE #include "bmInt.h" -void bmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonBmInfo *bmInfo) {} +static void bmGetMonitorInfo(SBnodeMgmt *pMgmt, SMonBmInfo *bmInfo) {} -int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { +int32_t bmProcessGetMonBmInfoReq(SBnodeMgmt *pMgmt, SNodeMsg *pReq) { SMonBmInfo bmInfo = {0}; - bmGetMonitorInfo(pWrapper, &bmInfo); - dmGetMonitorSysInfo(&bmInfo.sys); + bmGetMonitorInfo(pMgmt, &bmInfo); + dmGetMonitorSystemInfo(&bmInfo.sys); monGetLogs(&bmInfo.log); int32_t rspLen = tSerializeSMonBmInfo(NULL, 0, &bmInfo); @@ -43,8 +43,7 @@ int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { return 0; } -int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SDnode *pDnode = pWrapper->pDnode; +int32_t bmProcessCreateReq(const SMgmtInputOpt *pInput, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDCreateBnodeReq createReq = {0}; @@ -53,14 +52,14 @@ int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } - if (pDnode->data.dnodeId != 0 && createReq.dnodeId != pDnode->data.dnodeId) { + if (pInput->dnodeId != 0 && createReq.dnodeId != pInput->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; - dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->data.dnodeId); + dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pInput->dnodeId); return -1; } bool deployed = true; - if (dmWriteFile(pWrapper, deployed) != 0) { + if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { dError("failed to write bnode file since %s", terrstr()); return -1; } @@ -68,8 +67,7 @@ int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SDnode *pDnode = pWrapper->pDnode; +int32_t bmProcessDropReq(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDDropBnodeReq dropReq = {0}; @@ -78,14 +76,14 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } - if (dropReq.dnodeId != pDnode->data.dnodeId) { + if (pMgmt->dnodeId != 0 && dropReq.dnodeId != pMgmt->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; dError("failed to drop bnode since %s", terrstr()); return -1; } bool deployed = false; - if (dmWriteFile(pWrapper, deployed) != 0) { + if (dmWriteFile(pMgmt->path, pMgmt->name, deployed) != 0) { dError("failed to write bnode file since %s", terrstr()); return -1; } @@ -93,6 +91,19 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -void bmInitMsgHandle(SMgmtWrapper *pWrapper) { - dmSetMsgHandle(pWrapper, TDMT_MON_BM_INFO, bmProcessMonitorMsg, DEFAULT_HANDLE); +SArray *bmGetMsgHandles() { + int32_t code = -1; + SArray *pArray = taosArrayInit(2, sizeof(SMgmtHandle)); + if (pArray == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_MON_BM_INFO, bmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + + code = 0; +_OVER: + if (code != 0) { + taosArrayDestroy(pArray); + return NULL; + } else { + return pArray; + } } diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmInt.c b/source/dnode/mgmt/mgmt_bnode/src/bmInt.c index 2920d12eb4a4cf4c680f49f96eecdfe0a1f23302..45e0c3219330458420504354c94760f506e5e991 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmInt.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmInt.c @@ -16,18 +16,13 @@ #define _DEFAULT_SOURCE #include "bmInt.h" -static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dmReadFile(pWrapper, required); } - -static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { - SMsgCb msgCb = pMgmt->pDnode->data.msgCb; - msgCb.pWrapper = pMgmt->pWrapper; - pOption->msgCb = msgCb; +static int32_t bmRequire(const SMgmtInputOpt *pInput, bool *required) { + return dmReadFile(pInput->path, pInput->name, required); } -static void bmClose(SMgmtWrapper *pWrapper) { - SBnodeMgmt *pMgmt = pWrapper->pMgmt; - if (pMgmt == NULL) return; +static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { pOption->msgCb = pMgmt->msgCb; } +static void bmClose(SBnodeMgmt *pMgmt) { dInfo("bnode-mgmt start to cleanup"); if (pMgmt->pBnode != NULL) { bmStopWorker(pMgmt); @@ -35,12 +30,11 @@ static void bmClose(SMgmtWrapper *pWrapper) { pMgmt->pBnode = NULL; } - pWrapper->pMgmt = NULL; taosMemoryFree(pMgmt); dInfo("bnode-mgmt is cleaned up"); } -int32_t bmOpen(SMgmtWrapper *pWrapper) { +int32_t bmOpen(const SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dInfo("bnode-mgmt start to init"); SBnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SBnodeMgmt)); if (pMgmt == NULL) { @@ -48,40 +42,42 @@ int32_t bmOpen(SMgmtWrapper *pWrapper) { return -1; } - pMgmt->path = pWrapper->path; - pMgmt->pDnode = pWrapper->pDnode; - pMgmt->pWrapper = pWrapper; - pWrapper->pMgmt = pMgmt; + pMgmt->path = pInput->path; + pMgmt->name = pInput->name; + pMgmt->dnodeId = pInput->dnodeId; + pMgmt->msgCb = pInput->msgCb; + pMgmt->msgCb.pMgmt = pMgmt; SBnodeOpt option = {0}; bmInitOption(pMgmt, &option); pMgmt->pBnode = bndOpen(pMgmt->path, &option); if (pMgmt->pBnode == NULL) { dError("failed to open bnode since %s", terrstr()); - bmClose(pWrapper); + bmClose(pMgmt); return -1; } - dmReportStartup(pWrapper->pDnode, "bnode-impl", "initialized"); + tmsgReportStartup("bnode-impl", "initialized"); if (bmStartWorker(pMgmt) != 0) { dError("failed to start bnode worker since %s", terrstr()); - bmClose(pWrapper); + bmClose(pMgmt); return -1; } - dmReportStartup(pWrapper->pDnode, "bnode-worker", "initialized"); + tmsgReportStartup("bnode-worker", "initialized"); + pOutput->pMgmt = pMgmt; + dInfo("bnode-mgmt is initialized"); return 0; } -void bmSetMgmtFp(SMgmtWrapper *pWrapper) { - SMgmtFp mgmtFp = {0}; - mgmtFp.openFp = bmOpen; - mgmtFp.closeFp = bmClose; - mgmtFp.createFp = bmProcessCreateReq; - mgmtFp.dropFp = bmProcessDropReq; - mgmtFp.requiredFp = bmRequire; +SMgmtFunc bmGetMgmtFunc() { + SMgmtFunc mgmtFunc = {0}; + mgmtFunc.openFp = bmOpen; + mgmtFunc.closeFp = (NodeCloseFp)bmClose; + mgmtFunc.createFp = (NodeCreateFp)bmProcessCreateReq; + mgmtFunc.dropFp = (NodeDropFp)bmProcessDropReq; + mgmtFunc.requiredFp = bmRequire; + mgmtFunc.getHandlesFp = bmGetMsgHandles; - bmInitMsgHandle(pWrapper); - pWrapper->name = "bnode"; - pWrapper->fp = mgmtFp; + return mgmtFunc; } diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index d3204039e6769e62d28586da96aa9b1e3adf095a..1ab8b5ef3454928dc0d63024708c7de947cd54e4 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -58,7 +58,7 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { int32_t code = -1; if (pMsg->rpcMsg.msgType == TDMT_MON_BM_INFO) { - code = bmProcessGetMonBmInfoReq(pMgmt->pWrapper, pMsg); + code = bmProcessGetMonBmInfoReq(pMgmt, pMsg); } else { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } @@ -106,8 +106,7 @@ static void bmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO taosArrayDestroy(pArray); } -int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SBnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t bmPutNodeMsgToWriteQueue(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) { SMultiWorker *pWorker = &pMgmt->writeWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -115,8 +114,7 @@ int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SBnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t bmPutNodeMsgToMonitorQueue(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -136,18 +134,16 @@ int32_t bmStartWorker(SBnodeMgmt *pMgmt) { return -1; } - if (tsMultiProcess) { - SSingleWorkerCfg mCfg = { - .min = 1, - .max = 1, - .name = "bnode-monitor", - .fp = (FItem)bmProcessMonitorQueue, - .param = pMgmt, - }; - if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start bnode-monitor worker since %s", terrstr()); - return -1; - } + SSingleWorkerCfg mCfg = { + .min = 1, + .max = 1, + .name = "bnode-monitor", + .fp = (FItem)bmProcessMonitorQueue, + .param = pMgmt, + }; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start bnode-monitor worker since %s", terrstr()); + return -1; } dDebug("bnode workers are initialized"); diff --git a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a4268fc9f0d501ae377934d9df321984010c665e --- /dev/null +++ b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt @@ -0,0 +1,9 @@ +aux_source_directory(src MGMT_DNODE) +add_library(mgmt_dnode STATIC ${MGMT_DNODE}) +target_include_directories( + mgmt_dnode + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + mgmt_dnode node_util +) \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..5ef2706f1e9e94e282083b8483abc234be6a57a5 --- /dev/null +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_QNODE_INT_H_ +#define _TD_DND_QNODE_INT_H_ + +#include "dmUtil.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SDnodeMgmt { + struct SDnode *pDnode; + SMsgCb msgCb; + const char *path; + const char *name; + TdThread statusThread; + TdThread monitorThread; + SSingleWorker mgmtWorker; + ProcessCreateNodeFp processCreateNodeFp; + ProcessDropNodeFp processDropNodeFp; + IsNodeDeployedFp isNodeDeployedFp; + SDnodeData data; +} SDnodeMgmt; + +// dmEps.c +int32_t dmReadEps(SDnodeMgmt *pMgmt); +int32_t dmWriteEps(SDnodeMgmt *pMgmt); +void dmUpdateEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); + +// dmHandle.c +SArray *dmGetMsgHandles(); +void dmSendStatusReq(SDnodeMgmt *pMgmt); +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); + +// dmMonitor.c +void dmGetVnodeLoads(SDnodeMgmt *pMgmt, SMonVloadInfo *pInfo); +void dmGetMnodeLoads(SDnodeMgmt *pMgmt, SMonMloadInfo *pInfo); +void dmSendMonitorReport(SDnodeMgmt *pMgmt); + +// dmWorker.c +int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); +void dmStopStatusThread(SDnodeMgmt *pMgmt); +int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt); +void dmStopMonitorThread(SDnodeMgmt *pMgmt); +int32_t dmStartWorker(SDnodeMgmt *pMgmt); +void dmStopWorker(SDnodeMgmt *pMgmt); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_QNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/implement/src/dmEps.c b/source/dnode/mgmt/mgmt_dnode/src/dmEps.c similarity index 69% rename from source/dnode/mgmt/implement/src/dmEps.c rename to source/dnode/mgmt/mgmt_dnode/src/dmEps.c index f5c9a1d91b2fc7f2169386980ec627ac2f5473fc..9ebb02b9646c13fe8997f2899431e079d3fce128 100644 --- a/source/dnode/mgmt/implement/src/dmEps.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmEps.c @@ -14,16 +14,16 @@ */ #define _DEFAULT_SOURCE -#include "dmImp.h" +#include "dmInt.h" -static void dmPrintEps(SDnode *pDnode); -static bool dmIsEpChanged(SDnode *pDnode, int32_t dnodeId, const char *ep); -static void dmResetEps(SDnode *pDnode, SArray *dnodeEps); +static void dmPrintEps(SDnodeMgmt *pMgmt); +static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep); +static void dmResetEps(SDnodeMgmt *pMgmt, SArray *dnodeEps); -static void dmGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { - taosRLockLatch(&pDnode->data.latch); +static void dmGetDnodeEp(SDnodeMgmt *pMgmt, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { + taosRLockLatch(&pMgmt->data.latch); - SDnodeEp *pDnodeEp = taosHashGet(pDnode->data.dnodeHash, &dnodeId, sizeof(int32_t)); + SDnodeEp *pDnodeEp = taosHashGet(pMgmt->data.dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { if (pPort != NULL) { *pPort = pDnodeEp->ep.port; @@ -36,10 +36,10 @@ static void dmGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn } } - taosRUnLockLatch(&pDnode->data.latch); + taosRUnLockLatch(&pMgmt->data.latch); } -int32_t dmReadEps(SDnode *pDnode) { +int32_t dmReadEps(SDnodeMgmt *pMgmt) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 256 * 1024; @@ -48,16 +48,15 @@ int32_t dmReadEps(SDnode *pDnode) { char file[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - pDnode->data.dnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); - if (pDnode->data.dnodeEps == NULL) { + pMgmt->data.dnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); + if (pMgmt->data.dnodeEps == NULL) { dError("failed to calloc dnodeEp array since %s", strerror(errno)); goto _OVER; } - snprintf(file, sizeof(file), "%s%sdnode.json", pDnode->wrappers[DNODE].path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%sdnode.json", pMgmt->path, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - // dDebug("file %s not exist", file); code = 0; goto _OVER; } @@ -80,21 +79,21 @@ int32_t dmReadEps(SDnode *pDnode) { dError("failed to read %s since dnodeId not found", file); goto _OVER; } - pDnode->data.dnodeId = dnodeId->valueint; + pMgmt->data.dnodeId = dnodeId->valueint; cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); if (!clusterId || clusterId->type != cJSON_String) { dError("failed to read %s since clusterId not found", file); goto _OVER; } - pDnode->data.clusterId = atoll(clusterId->valuestring); + pMgmt->data.clusterId = atoll(clusterId->valuestring); cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); if (!dropped || dropped->type != cJSON_Number) { dError("failed to read %s since dropped not found", file); goto _OVER; } - pDnode->data.dropped = dropped->valueint; + pMgmt->data.dropped = dropped->valueint; cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); if (!dnodes || dnodes->type != cJSON_Array) { @@ -144,29 +143,29 @@ int32_t dmReadEps(SDnode *pDnode) { } dnodeEp.isMnode = isMnode->valueint; - taosArrayPush(pDnode->data.dnodeEps, &dnodeEp); + taosArrayPush(pMgmt->data.dnodeEps, &dnodeEp); } code = 0; dDebug("succcessed to read file %s", file); - dmPrintEps(pDnode); + dmPrintEps(pMgmt); _OVER: if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); - if (taosArrayGetSize(pDnode->data.dnodeEps) == 0) { + if (taosArrayGetSize(pMgmt->data.dnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; - taosGetFqdnPortFromEp(pDnode->data.firstEp, &dnodeEp.ep); - taosArrayPush(pDnode->data.dnodeEps, &dnodeEp); + taosGetFqdnPortFromEp(pMgmt->data.firstEp, &dnodeEp.ep); + taosArrayPush(pMgmt->data.dnodeEps, &dnodeEp); } - dmResetEps(pDnode, pDnode->data.dnodeEps); + dmResetEps(pMgmt, pMgmt->data.dnodeEps); - if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) { - dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file); + if (dmIsEpChanged(pMgmt, pMgmt->data.dnodeId, pMgmt->data.localEp)) { + dError("localEp %s different with %s and need reconfigured", pMgmt->data.localEp, file); return -1; } @@ -174,11 +173,11 @@ _OVER: return code; } -int32_t dmWriteEps(SDnode *pDnode) { +int32_t dmWriteEps(SDnodeMgmt *pMgmt) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%sdnode.json.bak", pDnode->wrappers[DNODE].path, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%sdnode.json", pDnode->wrappers[DNODE].path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%sdnode.json.bak", pMgmt->path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%sdnode.json", pMgmt->path, TD_DIRSEP); TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -192,14 +191,14 @@ int32_t dmWriteEps(SDnode *pDnode) { char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pDnode->data.dnodeId); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pDnode->data.clusterId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pDnode->data.dropped); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pMgmt->data.dnodeId); + len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pMgmt->data.clusterId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pMgmt->data.dropped); len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); - int32_t numOfEps = (int32_t)taosArrayGetSize(pDnode->data.dnodeEps); + int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->data.dnodeEps); for (int32_t i = 0; i < numOfEps; ++i) { - SDnodeEp *pDnodeEp = taosArrayGet(pDnode->data.dnodeEps, i); + SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->data.dnodeEps, i); len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); @@ -223,41 +222,41 @@ int32_t dmWriteEps(SDnode *pDnode) { return -1; } - pDnode->data.updateTime = taosGetTimestampMs(); + pMgmt->data.updateTime = taosGetTimestampMs(); dDebug("successed to write %s", realfile); return 0; } -void dmUpdateEps(SDnode *pDnode, SArray *eps) { +void dmUpdateEps(SDnodeMgmt *pMgmt, SArray *eps) { int32_t numOfEps = taosArrayGetSize(eps); if (numOfEps <= 0) return; - taosWLockLatch(&pDnode->data.latch); + taosWLockLatch(&pMgmt->data.latch); - int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pDnode->data.dnodeEps); + int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->data.dnodeEps); if (numOfEps != numOfEpsOld) { - dmResetEps(pDnode, eps); - dmWriteEps(pDnode); + dmResetEps(pMgmt, eps); + dmWriteEps(pMgmt); } else { int32_t size = numOfEps * sizeof(SDnodeEp); - if (memcmp(pDnode->data.dnodeEps->pData, eps->pData, size) != 0) { - dmResetEps(pDnode, eps); - dmWriteEps(pDnode); + if (memcmp(pMgmt->data.dnodeEps->pData, eps->pData, size) != 0) { + dmResetEps(pMgmt, eps); + dmWriteEps(pMgmt); } } - taosWUnLockLatch(&pDnode->data.latch); + taosWUnLockLatch(&pMgmt->data.latch); } -static void dmResetEps(SDnode *pDnode, SArray *dnodeEps) { - if (pDnode->data.dnodeEps != dnodeEps) { - SArray *tmp = pDnode->data.dnodeEps; - pDnode->data.dnodeEps = taosArrayDup(dnodeEps); +static void dmResetEps(SDnodeMgmt *pMgmt, SArray *dnodeEps) { + if (pMgmt->data.dnodeEps != dnodeEps) { + SArray *tmp = pMgmt->data.dnodeEps; + pMgmt->data.dnodeEps = taosArrayDup(dnodeEps); taosArrayDestroy(tmp); } - pDnode->data.mnodeEps.inUse = 0; - pDnode->data.mnodeEps.numOfEps = 0; + pMgmt->data.mnodeEps.inUse = 0; + pMgmt->data.mnodeEps.numOfEps = 0; int32_t mIndex = 0; int32_t numOfEps = (int32_t)taosArrayGetSize(dnodeEps); @@ -266,35 +265,35 @@ static void dmResetEps(SDnode *pDnode, SArray *dnodeEps) { SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); if (!pDnodeEp->isMnode) continue; if (mIndex >= TSDB_MAX_REPLICA) continue; - pDnode->data.mnodeEps.numOfEps++; + pMgmt->data.mnodeEps.numOfEps++; - pDnode->data.mnodeEps.eps[mIndex] = pDnodeEp->ep; + pMgmt->data.mnodeEps.eps[mIndex] = pDnodeEp->ep; mIndex++; } for (int32_t i = 0; i < numOfEps; i++) { SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); - taosHashPut(pDnode->data.dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); + taosHashPut(pMgmt->data.dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); } - dmPrintEps(pDnode); + dmPrintEps(pMgmt); } -static void dmPrintEps(SDnode *pDnode) { - int32_t numOfEps = (int32_t)taosArrayGetSize(pDnode->data.dnodeEps); +static void dmPrintEps(SDnodeMgmt *pMgmt) { + int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->data.dnodeEps); dDebug("print dnode ep list, num:%d", numOfEps); for (int32_t i = 0; i < numOfEps; i++) { - SDnodeEp *pEp = taosArrayGet(pDnode->data.dnodeEps, i); + SDnodeEp *pEp = taosArrayGet(pMgmt->data.dnodeEps, i); dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); } } -static bool dmIsEpChanged(SDnode *pDnode, int32_t dnodeId, const char *ep) { +static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep) { bool changed = false; if (dnodeId == 0) return changed; - taosRLockLatch(&pDnode->data.latch); + taosRLockLatch(&pMgmt->data.latch); - SDnodeEp *pDnodeEp = taosHashGet(pDnode->data.dnodeHash, &dnodeId, sizeof(int32_t)); + SDnodeEp *pDnodeEp = taosHashGet(pMgmt->data.dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { char epstr[TSDB_EP_LEN + 1] = {0}; snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); @@ -304,6 +303,6 @@ static bool dmIsEpChanged(SDnode *pDnode, int32_t dnodeId, const char *ep) { } } - taosRUnLockLatch(&pDnode->data.latch); + taosRUnLockLatch(&pMgmt->data.latch); return changed; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c new file mode 100644 index 0000000000000000000000000000000000000000..38f97d36006bf0d773e0d76a22fdf3a2902a6902 --- /dev/null +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { + if (pMgmt->data.dnodeId == 0 || pMgmt->data.clusterId == 0) { + dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); + taosWLockLatch(&pMgmt->data.latch); + pMgmt->data.dnodeId = pCfg->dnodeId; + pMgmt->data.clusterId = pCfg->clusterId; + dmWriteEps(pMgmt); + taosWUnLockLatch(&pMgmt->data.latch); + } +} + +static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { + if (pRsp->code != 0) { + if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->data.dropped && pMgmt->data.dnodeId > 0) { + dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->data.dnodeId); + pMgmt->data.dropped = 1; + dmWriteEps(pMgmt); + } + } else { + SStatusRsp statusRsp = {0}; + if (pRsp->pCont != NULL && pRsp->contLen > 0 && + tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { + pMgmt->data.dnodeVer = statusRsp.dnodeVer; + dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); + dmUpdateEps(pMgmt, statusRsp.pDnodeEps); + } + rpcFreeCont(pRsp->pCont); + tFreeSStatusRsp(&statusRsp); + } +} + +void dmSendStatusReq(SDnodeMgmt *pMgmt) { + SStatusReq req = {0}; + + taosRLockLatch(&pMgmt->data.latch); + req.sver = tsVersion; + req.dnodeVer = pMgmt->data.dnodeVer; + req.dnodeId = pMgmt->data.dnodeId; + req.clusterId = pMgmt->data.clusterId; + if (req.clusterId == 0) req.dnodeId = 0; + req.rebootTime = pMgmt->data.rebootTime; + req.updateTime = pMgmt->data.updateTime; + req.numOfCores = tsNumOfCores; + req.numOfSupportVnodes = pMgmt->data.supportVnodes; + tstrncpy(req.dnodeEp, pMgmt->data.localEp, TSDB_EP_LEN); + + req.clusterCfg.statusInterval = tsStatusInterval; + req.clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); + taosRUnLockLatch(&pMgmt->data.latch); + + SMonVloadInfo vinfo = {0}; + dmGetVnodeLoads(pMgmt, &vinfo); + req.pVloads = vinfo.pVloads; + pMgmt->data.unsyncedVgId = 0; + pMgmt->data.vndState = TAOS_SYNC_STATE_LEADER; + for (int32_t i = 0; i < taosArrayGetSize(req.pVloads); ++i) { + SVnodeLoad *pLoad = taosArrayGet(req.pVloads, i); + if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) { + pMgmt->data.unsyncedVgId = pLoad->vgId; + pMgmt->data.vndState = pLoad->syncState; + } + } + + SMonMloadInfo minfo = {0}; + dmGetMnodeLoads(pMgmt, &minfo); + pMgmt->data.mndState = minfo.load.syncState; + + int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); + void *pHead = rpcMallocCont(contLen); + tSerializeSStatusReq(pHead, contLen, &req); + tFreeSStatusReq(&req); + + SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)0x9527}; + SRpcMsg rpcRsp = {0}; + + dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); + tmsgSendMnodeRecv(&rpcMsg, &rpcRsp); + dmProcessStatusRsp(pMgmt, &rpcRsp); +} + +int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("auth rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pRsp = &pMsg->rpcMsg; + dError("grant rsp is received, but not supported yet"); + return 0; +} + +int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SRpcMsg *pReq = &pMsg->rpcMsg; + SDCfgDnodeReq *pCfg = pReq->pCont; + dError("config req is received, but not supported yet"); + return TSDB_CODE_OPS_NOT_SUPPORT; +} + +static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { + pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK; + pStatus->details[0] = 0; + + SServerStatusRsp statusRsp = {0}; + SMonMloadInfo minfo = {0}; + dmGetMnodeLoads(pMgmt, &minfo); + if (minfo.isMnode && minfo.load.syncState != TAOS_SYNC_STATE_LEADER && + minfo.load.syncState != TAOS_SYNC_STATE_CANDIDATE) { + pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; + snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); + return; + } + + SMonVloadInfo vinfo = {0}; + dmGetVnodeLoads(pMgmt, &vinfo); + for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { + SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); + if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) { + pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; + snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId, + syncStr(pLoad->syncState)); + break; + } + } + + taosArrayDestroy(vinfo.pVloads); +} + +int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dDebug("server run status req is received"); + SServerStatusRsp statusRsp = {0}; + dmGetServerRunStatus(pMgmt, &statusRsp); + + SRpcMsg rspMsg = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .refId = pMsg->rpcMsg.refId}; + int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); + if (rspLen < 0) { + rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp); + pMsg->pRsp = pRsp; + pMsg->rspLen = rspLen; + return 0; +} + +SArray *dmGetMsgHandles() { + int32_t code = -1; + SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle)); + if (pArray == NULL) goto _OVER; + + // Requests handled by DNODE + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + + // Requests handled by MNODE + if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + + code = 0; + +_OVER: + if (code != 0) { + taosArrayDestroy(pArray); + return NULL; + } else { + return pArray; + } +} diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c new file mode 100644 index 0000000000000000000000000000000000000000..9e40b3d02253506839746dd9c42229eda2cf6594 --- /dev/null +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { + if (dmStartStatusThread(pMgmt) != 0) { + return -1; + } + if (dmStartMonitorThread(pMgmt) != 0) { + return -1; + } + return 0; +} + +static void dmStopMgmt(SDnodeMgmt *pMgmt) { + pMgmt->data.stopped = true; + dmStopMonitorThread(pMgmt); + dmStopStatusThread(pMgmt); +} + +static int32_t dmOpenMgmt(const SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { + dInfo("dnode-mgmt start to init"); + SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); + if (pMgmt == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMgmt->data.dnodeId = 0; + pMgmt->data.clusterId = 0; + pMgmt->data.dnodeVer = 0; + pMgmt->data.updateTime = 0; + pMgmt->data.rebootTime = taosGetTimestampMs(); + pMgmt->data.dropped = 0; + pMgmt->data.localEp = pInput->localEp; + pMgmt->data.localFqdn = pInput->localFqdn; + pMgmt->data.firstEp = pInput->firstEp; + pMgmt->data.secondEp = pInput->secondEp; + pMgmt->data.supportVnodes = pInput->supportVnodes; + pMgmt->data.serverPort = pInput->serverPort; + pMgmt->pDnode = pInput->pDnode; + pMgmt->msgCb = pInput->msgCb; + pMgmt->path = pInput->path; + pMgmt->name = pInput->name; + pMgmt->processCreateNodeFp = pInput->processCreateNodeFp; + pMgmt->processDropNodeFp = pInput->processDropNodeFp; + pMgmt->isNodeDeployedFp = pInput->isNodeDeployedFp; + taosInitRWLatch(&pMgmt->data.latch); + + pMgmt->data.dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMgmt->data.dnodeHash == NULL) { + dError("failed to init dnode hash"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (dmReadEps(pMgmt) != 0) { + dError("failed to read file since %s", terrstr()); + return -1; + } + + if (pMgmt->data.dropped) { + dError("dnode will not start since its already dropped"); + return -1; + } + + if (dmStartWorker(pMgmt) != 0) { + return -1; + } + + if (udfStartUdfd(pMgmt->data.dnodeId) != 0) { + dError("failed to start udfd"); + } + + pOutput->pMgmt = pMgmt; + pOutput->mnodeEps = pMgmt->data.mnodeEps; + dInfo("dnode-mgmt is initialized"); + return 0; +} + +static void dmCloseMgmt(SDnodeMgmt *pMgmt) { + dInfo("dnode-mgmt start to clean up"); + dmStopWorker(pMgmt); + + taosWLockLatch(&pMgmt->data.latch); + if (pMgmt->data.dnodeEps != NULL) { + taosArrayDestroy(pMgmt->data.dnodeEps); + pMgmt->data.dnodeEps = NULL; + } + if (pMgmt->data.dnodeHash != NULL) { + taosHashCleanup(pMgmt->data.dnodeHash); + pMgmt->data.dnodeHash = NULL; + } + taosWUnLockLatch(&pMgmt->data.latch); + taosMemoryFree(pMgmt); + + dInfo("dnode-mgmt is cleaned up"); +} + +static int32_t dmRequireMgmt(const SMgmtInputOpt *pInput, bool *required) { + *required = true; + return 0; +} + +SMgmtFunc dmGetMgmtFunc() { + SMgmtFunc mgmtFunc = {0}; + mgmtFunc.openFp = dmOpenMgmt; + mgmtFunc.closeFp = (NodeCloseFp)dmCloseMgmt; + mgmtFunc.startFp = (NodeStartFp)dmStartMgmt; + mgmtFunc.stopFp = (NodeStopFp)dmStopMgmt; + mgmtFunc.requiredFp = dmRequireMgmt; + mgmtFunc.getHandlesFp = dmGetMsgHandles; + + return mgmtFunc; +} diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c b/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c new file mode 100644 index 0000000000000000000000000000000000000000..d2f16c3ad0ba41d3a6bf20881fdd0ffbdda3e8a7 --- /dev/null +++ b/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +#define dmSendLocalRecv(pMgmt, mtype, func, pInfo) \ + if (!tsMultiProcess) { \ + SRpcMsg rsp = {0}; \ + SRpcMsg req = {.msgType = mtype}; \ + SEpSet epset = {.inUse = 0, .numOfEps = 1}; \ + tstrncpy(epset.eps[0].fqdn, pMgmt->data.localFqdn, TSDB_FQDN_LEN); \ + epset.eps[0].port = pMgmt->data.serverPort; \ + \ + rpcSendRecv(pMgmt->msgCb.clientRpc, &epset, &req, &rsp); \ + if (rsp.code == 0 && rsp.contLen > 0) { \ + func(rsp.pCont, rsp.contLen, pInfo); \ + } \ + rpcFreeCont(rsp.pCont); \ + } + +static void dmGetMonitorBasicInfo(SDnodeMgmt *pMgmt, SMonBasicInfo *pInfo) { + pInfo->protocol = 1; + pInfo->dnode_id = pMgmt->data.dnodeId; + pInfo->cluster_id = pMgmt->data.clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + +static void dmGetMonitorDnodeInfo(SDnodeMgmt *pMgmt, SMonDnodeInfo *pInfo) { + pInfo->uptime = (taosGetTimestampMs() - pMgmt->data.rebootTime) / (86400000.0f); + pInfo->has_mnode = (*pMgmt->isNodeDeployedFp)(pMgmt->pDnode, MNODE); + pInfo->has_qnode = (*pMgmt->isNodeDeployedFp)(pMgmt->pDnode, QNODE); + pInfo->has_snode = (*pMgmt->isNodeDeployedFp)(pMgmt->pDnode, SNODE); + pInfo->has_bnode = (*pMgmt->isNodeDeployedFp)(pMgmt->pDnode, BNODE); + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; +} + +static void dmGetMonitorInfo(SDnodeMgmt *pMgmt, SMonDmInfo *pInfo) { + dmGetMonitorBasicInfo(pMgmt, &pInfo->basic); + dmGetMonitorDnodeInfo(pMgmt, &pInfo->dnode); + dmGetMonitorSystemInfo(&pInfo->sys); +} + +void dmSendMonitorReport(SDnodeMgmt *pMgmt) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SMonDmInfo dmInfo = {0}; + SMonMmInfo mmInfo = {0}; + SMonVmInfo vmInfo = {0}; + SMonQmInfo qmInfo = {0}; + SMonSmInfo smInfo = {0}; + SMonBmInfo bmInfo = {0}; + + dmGetMonitorInfo(pMgmt, &dmInfo); + dmSendLocalRecv(pMgmt, TDMT_MON_VM_INFO, tDeserializeSMonVmInfo, &vmInfo); + if (dmInfo.dnode.has_mnode) { + dmSendLocalRecv(pMgmt, TDMT_MON_MM_INFO, tDeserializeSMonMmInfo, &mmInfo); + } + if (dmInfo.dnode.has_qnode) { + dmSendLocalRecv(pMgmt, TDMT_MON_QM_INFO, tDeserializeSMonQmInfo, &qmInfo); + } + if (dmInfo.dnode.has_snode) { + dmSendLocalRecv(pMgmt, TDMT_MON_SM_INFO, tDeserializeSMonSmInfo, &smInfo); + } + if (dmInfo.dnode.has_bnode) { + dmSendLocalRecv(pMgmt, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); + } + + monSetDmInfo(&dmInfo); + monSetMmInfo(&mmInfo); + monSetVmInfo(&vmInfo); + monSetQmInfo(&qmInfo); + monSetSmInfo(&smInfo); + monSetBmInfo(&bmInfo); + tFreeSMonMmInfo(&mmInfo); + tFreeSMonVmInfo(&vmInfo); + tFreeSMonQmInfo(&qmInfo); + tFreeSMonSmInfo(&smInfo); + tFreeSMonBmInfo(&bmInfo); + monSendReport(); +} + +void dmGetVnodeLoads(SDnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { + dmSendLocalRecv(pMgmt, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo); +} + +void dmGetMnodeLoads(SDnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { + dmSendLocalRecv(pMgmt, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); +} diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c new file mode 100644 index 0000000000000000000000000000000000000000..af02e56f0569257b714edd696bac56a0ae877891 --- /dev/null +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmInt.h" + +static void *dmStatusThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + int64_t lastTime = taosGetTimestampMs(); + + setThreadName("dnode-status"); + + while (1) { + taosMsleep(200); + if (pMgmt->data.dropped || pMgmt->data.stopped) break; + + int64_t curTime = taosGetTimestampMs(); + float interval = (curTime - lastTime) / 1000.0f; + if (interval >= tsStatusInterval) { + dmSendStatusReq(pMgmt); + lastTime = curTime; + } + } + + return NULL; +} + +static void *dmMonitorThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + int64_t lastTime = taosGetTimestampMs(); + + setThreadName("dnode-monitor"); + + while (1) { + taosMsleep(200); + if (pMgmt->data.dropped || pMgmt->data.stopped) break; + + int64_t curTime = taosGetTimestampMs(); + float interval = (curTime - lastTime) / 1000.0f; + if (interval >= tsMonitorInterval) { + dmSendMonitorReport(pMgmt); + lastTime = curTime; + } + } + + return NULL; +} + +int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) { + dError("failed to create status thread since %s", strerror(errno)); + return -1; + } + + taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("dnode-status", "initialized"); + return 0; +} + +void dmStopStatusThread(SDnodeMgmt *pMgmt) { + if (taosCheckPthreadValid(pMgmt->statusThread)) { + taosThreadJoin(pMgmt->statusThread, NULL); + } +} + +int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->monitorThread, &thAttr, dmMonitorThreadFp, pMgmt) != 0) { + dError("failed to create monitor thread since %s", strerror(errno)); + return -1; + } + + taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("dnode-monitor", "initialized"); + return 0; +} + +void dmStopMonitorThread(SDnodeMgmt *pMgmt) { + if (taosCheckPthreadValid(pMgmt->monitorThread)) { + taosThreadJoin(pMgmt->monitorThread, NULL); + } +} + +static void dmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SDnodeMgmt *pMgmt = pInfo->ahandle; + int32_t code = -1; + tmsg_t msgType = pMsg->rpcMsg.msgType; + bool isRequest = msgType & 1u; + dTrace("msg:%p, will be processed in dnode-mgmt queue, type:%s", pMsg, TMSG_INFO(msgType)); + + switch (msgType) { + case TDMT_DND_CONFIG_DNODE: + code = dmProcessConfigReq(pMgmt, pMsg); + break; + case TDMT_MND_AUTH_RSP: + code = dmProcessAuthRsp(pMgmt, pMsg); + break; + case TDMT_MND_GRANT_RSP: + code = dmProcessGrantRsp(pMgmt, pMsg); + break; + case TDMT_DND_CREATE_MNODE: + code = (*pMgmt->processCreateNodeFp)(pMgmt->pDnode, MNODE, pMsg); + break; + case TDMT_DND_DROP_MNODE: + code = (*pMgmt->processDropNodeFp)(pMgmt->pDnode, MNODE, pMsg); + break; + case TDMT_DND_CREATE_QNODE: + code = (*pMgmt->processCreateNodeFp)(pMgmt->pDnode, QNODE, pMsg); + break; + case TDMT_DND_DROP_QNODE: + code = (*pMgmt->processDropNodeFp)(pMgmt->pDnode, QNODE, pMsg); + break; + case TDMT_DND_CREATE_SNODE: + code = (*pMgmt->processCreateNodeFp)(pMgmt->pDnode, SNODE, pMsg); + break; + case TDMT_DND_DROP_SNODE: + code = (*pMgmt->processDropNodeFp)(pMgmt->pDnode, SNODE, pMsg); + break; + case TDMT_DND_CREATE_BNODE: + code = (*pMgmt->processCreateNodeFp)(pMgmt->pDnode, BNODE, pMsg); + break; + case TDMT_DND_DROP_BNODE: + code = (*pMgmt->processDropNodeFp)(pMgmt->pDnode, BNODE, pMsg); + break; + case TDMT_DND_SERVER_STATUS: + code = dmProcessServerRunStatus(pMgmt, pMsg); + break; + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + break; + } + + if (isRequest) { + if (code != 0 && terrno != 0) code = terrno; + SRpcMsg rsp = { + .handle = pMsg->rpcMsg.handle, + .ahandle = pMsg->rpcMsg.ahandle, + .code = code, + .refId = pMsg->rpcMsg.refId, + .pCont = pMsg->pRsp, + .contLen = pMsg->rspLen, + }; + rpcSendResponse(&rsp); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +int32_t dmStartWorker(SDnodeMgmt *pMgmt) { + SSingleWorkerCfg cfg = { + .min = 1, + .max = 1, + .name = "dnode-mgmt", + .fp = (FItem)dmProcessMgmtQueue, + .param = pMgmt, + }; + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) { + dError("failed to start dnode-mgmt worker since %s", terrstr()); + return -1; + } + + dDebug("dnode workers are initialized"); + return 0; +} + +void dmStopWorker(SDnodeMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->mgmtWorker); + dDebug("dnode workers are closed"); +} + +int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { + SSingleWorker *pWorker = &pMgmt->mgmtWorker; + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} diff --git a/source/dnode/mgmt/mgmt_mnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_mnode/CMakeLists.txt index 5ca9af5628cb9bc54b8af860b9efdc753cd93121..04118590a2ef0fce1f1a6d38e3862a41a0f9ae32 100644 --- a/source/dnode/mgmt/mgmt_mnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_mnode/CMakeLists.txt @@ -5,5 +5,5 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mgmt_mnode dnode_interface + mgmt_mnode node_util ) \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 4d40d1fa287417aff15600ae7dc56618c44a3cb0..87dbe702beb45948d61bd44025ab3d3363c801e6 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -16,8 +16,7 @@ #ifndef _TD_DND_MNODE_INT_H_ #define _TD_DND_MNODE_INT_H_ -#include "dmInt.h" - +#include "dmUtil.h" #include "mnode.h" #ifdef __cplusplus @@ -26,9 +25,10 @@ extern "C" { typedef struct SMnodeMgmt { SMnode *pMnode; - SDnode *pDnode; - SMgmtWrapper *pWrapper; + SMsgCb msgCb; const char *path; + const char *name; + int32_t dnodeId; SSingleWorker queryWorker; SSingleWorker readWorker; SSingleWorker writeWorker; @@ -41,33 +41,31 @@ typedef struct SMnodeMgmt { // mmFile.c int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed); -int32_t mmWriteFile(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq, bool deployed); +int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq, bool deployed); // mmInt.c int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq); // mmHandle.c -void mmInitMsgHandle(SMgmtWrapper *pWrapper); -int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +SArray *mmGetMsgHandles(); +int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SNodeMsg *pMsg); +int32_t mmProcessDropReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); -int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); -int32_t mmProcessGetMnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); -void mmGetMnodeLoads(SMgmtWrapper *pWrapper, SMonMloadInfo *pInfo); +int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SNodeMsg *pReq); +int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SNodeMsg *pReq); // mmWorker.c int32_t mmStartWorker(SMnodeMgmt *pMgmt); void mmStopWorker(SMnodeMgmt *pMgmt); -int32_t mmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t mmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t mmProcessReadMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); - -int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); -int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); -int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); -int32_t mmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); +int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc); +int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc); +int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc); +int32_t mmPutRpcMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 83c832a41eca6f220e1de218cad4bd88e8c3554d..df377fefe796960c6d793c7b3550ccec8c4e50c3 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -28,7 +28,6 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - // dDebug("file %s not exist", file); code = 0; goto _OVER; } @@ -105,11 +104,11 @@ _OVER: return code; } -int32_t mmWriteFile(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq, bool deployed) { +int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq, bool deployed) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%smnode.json.bak", pWrapper->path, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%smnode.json", pWrapper->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -125,22 +124,19 @@ int32_t mmWriteFile(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq, bool deploye len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - if (pReq != NULL || pMgmt != NULL) { - int8_t replica = (pReq != NULL ? pReq->replica : pMgmt->replica); - for (int32_t i = 0; i < replica; ++i) { - SReplica *pReplica = &pMgmt->replicas[i]; - if (pReq != NULL) { - pReplica = &pReq->replicas[i]; - } - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); - if (i < replica - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }],\n"); - } + int8_t replica = (pReq != NULL ? pReq->replica : pMgmt->replica); + for (int32_t i = 0; i < replica; ++i) { + SReplica *pReplica = &pMgmt->replicas[i]; + if (pReq != NULL) { + pReplica = &pReq->replicas[i]; + } + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); + if (i < replica - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }],\n"); } } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index ed9384a869562535045f8bfc9e718530854d8372..5548a23c55210b47c4b2147239228a71e99bb8ce 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -16,15 +16,14 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; +static void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *mmInfo) { mndGetMonitorInfo(pMgmt->pMnode, &mmInfo->cluster, &mmInfo->vgroup, &mmInfo->grant); } -int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { +int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SNodeMsg *pReq) { SMonMmInfo mmInfo = {0}; - mmGetMonitorInfo(pWrapper, &mmInfo); - dmGetMonitorSysInfo(&mmInfo.sys); + mmGetMonitorInfo(pMgmt, &mmInfo); + dmGetMonitorSystemInfo(&mmInfo.sys); monGetLogs(&mmInfo.log); int32_t rspLen = tSerializeSMonMmInfo(NULL, 0, &mmInfo); @@ -46,15 +45,14 @@ int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { return 0; } -void mmGetMnodeLoads(SMgmtWrapper *pWrapper, SMonMloadInfo *pInfo) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; +static void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { pInfo->isMnode = 1; mndGetLoad(pMgmt->pMnode, &pInfo->load); } -int32_t mmProcessGetMnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { +int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SNodeMsg *pReq) { SMonMloadInfo mloads = {0}; - mmGetMnodeLoads(pWrapper, &mloads); + mmGetMnodeLoads(pMgmt, &mloads); int32_t rspLen = tSerializeSMonMloadInfo(NULL, 0, &mloads); if (rspLen < 0) { @@ -74,8 +72,7 @@ int32_t mmProcessGetMnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { return 0; } -int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SDnode *pDnode = pWrapper->pDnode; +int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDCreateMnodeReq createReq = {0}; @@ -84,14 +81,18 @@ int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } - if (createReq.replica <= 1 || createReq.dnodeId != pDnode->data.dnodeId) { + if (createReq.replica <= 1 || (createReq.dnodeId != pInput->dnodeId && pInput->dnodeId != 0)) { terrno = TSDB_CODE_INVALID_OPTION; dError("failed to create mnode since %s", terrstr()); return -1; } bool deployed = true; - if (mmWriteFile(pWrapper, &createReq, deployed) != 0) { + + SMnodeMgmt mgmt = {0}; + mgmt.path = pInput->path; + mgmt.name = pInput->name; + if (mmWriteFile(&mgmt, &createReq, deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } @@ -99,8 +100,7 @@ int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SDnode *pDnode = pWrapper->pDnode; +int32_t mmProcessDropReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDDropMnodeReq dropReq = {0}; @@ -109,14 +109,14 @@ int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } - if (dropReq.dnodeId != pDnode->data.dnodeId) { + if (pMgmt->dnodeId != 0 && dropReq.dnodeId != pMgmt->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; dError("failed to drop mnode since %s", terrstr()); return -1; } bool deployed = false; - if (mmWriteFile(pWrapper, NULL, deployed) != 0) { + if (mmWriteFile(pMgmt, NULL, deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } @@ -125,7 +125,6 @@ int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { - SDnode *pDnode = pMgmt->pDnode; SRpcMsg *pReq = &pMsg->rpcMsg; SDAlterMnodeReq alterReq = {0}; @@ -134,104 +133,118 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { return -1; } - if (pDnode->data.dnodeId != 0 && alterReq.dnodeId != pDnode->data.dnodeId) { + if (pMgmt->dnodeId != 0 && alterReq.dnodeId != pMgmt->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; - dError("failed to alter mnode since %s, input:%d cur:%d", terrstr(), alterReq.dnodeId, pDnode->data.dnodeId); + dError("failed to alter mnode since %s, input:%d cur:%d", terrstr(), alterReq.dnodeId, pMgmt->dnodeId); return -1; } else { return mmAlter(pMgmt, &alterReq); } } -void mmInitMsgHandle(SMgmtWrapper *pWrapper) { - dmSetMsgHandle(pWrapper, TDMT_MON_MM_INFO, mmProcessMonitorMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MON_MM_LOAD, mmProcessMonitorMsg, DEFAULT_HANDLE); +SArray *mmGetMsgHandles() { + int32_t code = -1; + SArray *pArray = taosArrayInit(64, sizeof(SMgmtHandle)); + if (pArray == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; // Requests handled by DNODE - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; // Requests handled by MNODE - dmSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_QNODE_LIST, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_MQ_ASK_EP, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE); + if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_QNODE_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; // Requests handled by VNODE - dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE); - - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + + code = 0; + +_OVER: + if (code != 0) { + taosArrayDestroy(pArray); + return NULL; + } else { + return pArray; + } } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 0bf846b7fc772952965a43ca2fd979a85bd661a3..8815af2f22944e7495ad7afe0bfcf243e7e65f0c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -17,45 +17,35 @@ #include "mmInt.h" #include "wal.h" -static bool mmDeployRequired(SDnode *pDnode) { - if (pDnode->data.dnodeId > 0) return false; - if (pDnode->data.clusterId > 0) return false; - if (strcmp(pDnode->data.localEp, pDnode->data.firstEp) != 0) return false; +static bool mmDeployRequired(const SMgmtInputOpt *pInput) { + if (pInput->dnodeId > 0) return false; + if (pInput->clusterId > 0) return false; + if (strcmp(pInput->localEp, pInput->firstEp) != 0) return false; return true; } -static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) { +static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { SMnodeMgmt mgmt = {0}; - mgmt.path = pWrapper->path; + mgmt.path = pInput->path; if (mmReadFile(&mgmt, required) != 0) { return -1; } if (!(*required)) { - *required = mmDeployRequired(pWrapper->pDnode); + *required = mmDeployRequired(pInput); } return 0; } -static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { - SMsgCb msgCb = pMgmt->pDnode->data.msgCb; - msgCb.pWrapper = pMgmt->pWrapper; - msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToQueryQueue; - msgCb.queueFps[READ_QUEUE] = mmPutMsgToReadQueue; - msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue; - msgCb.queueFps[SYNC_QUEUE] = mmPutMsgToWriteQueue; - pOption->msgCb = msgCb; -} - -static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { - mmInitOption(pMgmt, pOption); +static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { + pOption->msgCb = pMgmt->msgCb; pOption->replica = 1; pOption->selfIndex = 0; SReplica *pReplica = &pOption->replicas[0]; pReplica->id = 1; - pReplica->port = pMgmt->pDnode->data.serverPort; - tstrncpy(pReplica->fqdn, pMgmt->pDnode->data.localFqdn, TSDB_FQDN_LEN); + pReplica->port = pInput->serverPort; + tstrncpy(pReplica->fqdn, pInput->localFqdn, TSDB_FQDN_LEN); pOption->deploy = true; pMgmt->selfIndex = pOption->selfIndex; @@ -64,7 +54,7 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { - mmInitOption(pMgmt, pOption); + pOption->msgCb = pMgmt->msgCb; pOption->selfIndex = pMgmt->selfIndex; pOption->replica = pMgmt->replica; memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); @@ -72,8 +62,7 @@ static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { } static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) { - mmInitOption(pMgmt, pOption); - + pOption->msgCb = pMgmt->msgCb; pOption->replica = pCreate->replica; pOption->selfIndex = -1; for (int32_t i = 0; i < pCreate->replica; ++i) { @@ -81,7 +70,7 @@ static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCre pReplica->id = pCreate->replicas[i].id; pReplica->port = pCreate->replicas[i].port; memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); - if (pReplica->id == pMgmt->pDnode->data.dnodeId) { + if (pReplica->id == pMgmt->dnodeId) { pOption->selfIndex = i; } } @@ -109,7 +98,7 @@ int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq) { } bool deployed = true; - if (mmWriteFile(pMgmt->pWrapper, pReq, deployed) != 0) { + if (mmWriteFile(pMgmt, pReq, deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } @@ -117,10 +106,7 @@ int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq) { return 0; } -static void mmClose(SMgmtWrapper *pWrapper) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - if (pMgmt == NULL) return; - +static void mmClose(SMnodeMgmt *pMgmt) { dInfo("mnode-mgmt start to cleanup"); if (pMgmt->pMnode != NULL) { mmStopWorker(pMgmt); @@ -128,12 +114,11 @@ static void mmClose(SMgmtWrapper *pWrapper) { pMgmt->pMnode = NULL; } - pWrapper->pMgmt = NULL; taosMemoryFree(pMgmt); dInfo("mnode-mgmt is cleaned up"); } -static int32_t mmOpen(SMgmtWrapper *pWrapper) { +static int32_t mmOpen(const SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dInfo("mnode-mgmt start to init"); if (walInit() != 0) { dError("failed to init wal since %s", terrstr()); @@ -146,23 +131,28 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) { return -1; } - pMgmt->path = pWrapper->path; - pMgmt->pDnode = pWrapper->pDnode; - pMgmt->pWrapper = pWrapper; - pWrapper->pMgmt = pMgmt; + pMgmt->path = pInput->path; + pMgmt->name = pInput->name; + pMgmt->dnodeId = pInput->dnodeId; + pMgmt->msgCb = pInput->msgCb; + pMgmt->msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)mmPutRpcMsgToQueryQueue; + pMgmt->msgCb.queueFps[READ_QUEUE] = (PutToQueueFp)mmPutRpcMsgToReadQueue; + pMgmt->msgCb.queueFps[WRITE_QUEUE] = (PutToQueueFp)mmPutRpcMsgToWriteQueue; + pMgmt->msgCb.queueFps[SYNC_QUEUE] = (PutToQueueFp)mmPutRpcMsgToWriteQueue; + pMgmt->msgCb.pMgmt = pMgmt; bool deployed = false; if (mmReadFile(pMgmt, &deployed) != 0) { dError("failed to read file since %s", terrstr()); - mmClose(pWrapper); + mmClose(pMgmt); return -1; } SMnodeOpt option = {0}; if (!deployed) { dInfo("mnode start to deploy"); - pWrapper->pDnode->data.dnodeId = 1; - mmBuildOptionForDeploy(pMgmt, &option); + pMgmt->dnodeId = 1; + mmBuildOptionForDeploy(pMgmt, pInput, &option); } else { dInfo("mnode start to open"); mmBuildOptionForOpen(pMgmt, &option); @@ -171,55 +161,52 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) { pMgmt->pMnode = mndOpen(pMgmt->path, &option); if (pMgmt->pMnode == NULL) { dError("failed to open mnode since %s", terrstr()); - mmClose(pWrapper); + mmClose(pMgmt); return -1; } - dmReportStartup(pWrapper->pDnode, "mnode-impl", "initialized"); + tmsgReportStartup("mnode-impl", "initialized"); if (mmStartWorker(pMgmt) != 0) { dError("failed to start mnode worker since %s", terrstr()); - mmClose(pWrapper); + mmClose(pMgmt); return -1; } - dmReportStartup(pWrapper->pDnode, "mnode-worker", "initialized"); + tmsgReportStartup("mnode-worker", "initialized"); if (!deployed) { deployed = true; - if (mmWriteFile(pWrapper, NULL, deployed) != 0) { + if (mmWriteFile(pMgmt, NULL, deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } } + pOutput->dnodeId = pMgmt->dnodeId; + pOutput->pMgmt = pMgmt; dInfo("mnode-mgmt is initialized"); return 0; } -static int32_t mmStart(SMgmtWrapper *pWrapper) { +static int32_t mmStart(SMnodeMgmt *pMgmt) { dDebug("mnode-mgmt start to run"); - SMnodeMgmt *pMgmt = pWrapper->pMgmt; return mndStart(pMgmt->pMnode); } -static void mmStop(SMgmtWrapper *pWrapper) { +static void mmStop(SMnodeMgmt *pMgmt) { dDebug("mnode-mgmt start to stop"); - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - if (pMgmt != NULL) { - mndStop(pMgmt->pMnode); - } + mndStop(pMgmt->pMnode); } -void mmSetMgmtFp(SMgmtWrapper *pWrapper) { - SMgmtFp mgmtFp = {0}; - mgmtFp.openFp = mmOpen; - mgmtFp.closeFp = mmClose; - mgmtFp.startFp = mmStart; - mgmtFp.stopFp = mmStop; - mgmtFp.createFp = mmProcessCreateReq; - mgmtFp.dropFp = mmProcessDropReq; - mgmtFp.requiredFp = mmRequire; - - mmInitMsgHandle(pWrapper); - pWrapper->name = "mnode"; - pWrapper->fp = mgmtFp; +SMgmtFunc mmGetMgmtFunc() { + SMgmtFunc mgmtFunc = {0}; + mgmtFunc.openFp = mmOpen; + mgmtFunc.closeFp = (NodeCloseFp)mmClose; + mgmtFunc.startFp = (NodeStartFp)mmStart; + mgmtFunc.stopFp = (NodeStopFp)mmStop; + mgmtFunc.createFp = (NodeCreateFp)mmProcessCreateReq; + mgmtFunc.dropFp = (NodeDropFp)mmProcessDropReq; + mgmtFunc.requiredFp = mmRequire; + mgmtFunc.getHandlesFp = mmGetMsgHandles; + + return mgmtFunc; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index aac5bbc16afdc074d785a15f836f6c9d637bc1ce..622b8332fce15a61af82d3bcd64612dcd40960fe 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -30,27 +30,27 @@ static inline void mmSendRsp(SNodeMsg *pMsg, int32_t code) { static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; - - int32_t code = -1; - tmsg_t msgType = pMsg->rpcMsg.msgType; - dTrace("msg:%p, get from mnode queue", pMsg); + int32_t code = -1; + tmsg_t msgType = pMsg->rpcMsg.msgType; + bool isRequest = msgType & 1U; + dTrace("msg:%p, get from mnode queue, type:%s", pMsg, TMSG_INFO(msgType)); switch (msgType) { case TDMT_DND_ALTER_MNODE: code = mmProcessAlterReq(pMgmt, pMsg); break; case TDMT_MON_MM_INFO: - code = mmProcessGetMonMmInfoReq(pMgmt->pWrapper, pMsg); + code = mmProcessGetMonitorInfoReq(pMgmt, pMsg); break; case TDMT_MON_MM_LOAD: - code = mmProcessGetMnodeLoadsReq(pMgmt->pWrapper, pMsg); + code = mmProcessGetLoadsReq(pMgmt, pMsg); break; default: pMsg->pNode = pMgmt->pMnode; code = mndProcessMsg(pMsg); } - if (msgType & 1U) { + if (isRequest) { if (pMsg->rpcMsg.handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { if (code != 0 && terrno != 0) code = terrno; mmSendRsp(pMsg, code); @@ -64,63 +64,47 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; - + int32_t code = -1; + tmsg_t msgType = pMsg->rpcMsg.msgType; + bool isRequest = msgType & 1U; dTrace("msg:%p, get from mnode-query queue", pMsg); - SRpcMsg *pRpc = &pMsg->rpcMsg; - int32_t code = -1; pMsg->pNode = pMgmt->pMnode; code = mndProcessMsg(pMsg); - if (pRpc->msgType & 1U) { - if (pRpc->handle != NULL && code != 0) { - dError("msg:%p, failed to process since %s", pMsg, terrstr()); + if (isRequest) { + if (pMsg->rpcMsg.handle != NULL && code != 0) { + if (code != 0 && terrno != 0) code = terrno; mmSendRsp(pMsg, code); } } dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - rpcFreeCont(pRpc->pCont); + rpcFreeCont(pMsg->rpcMsg.pCont); taosFreeQitem(pMsg); } -static void mmPutNodeMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) { - dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); +static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) { + dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->rpcMsg.msgType)); taosWriteQitem(pWorker->queue, pMsg); -} - -int32_t mmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); return 0; } -int32_t mmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); - return 0; -} +int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); } -int32_t mmProcessReadMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); - return 0; -} +int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); } -int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); - return 0; +int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); } + +int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); } -int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - mmPutNodeMsgToWorker(&pMgmt->monitorWorker, pMsg); - return 0; +int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->monitorWorker, pMsg); } -static int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) { - SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); +static inline int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) { + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg), RPC_QITEM); if (pMsg == NULL) return -1; dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); @@ -129,25 +113,19 @@ static int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) { return 0; } -int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pRpc); } -int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pRpc); } -int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(&pMgmt->readWorker, pRpc); } -int32_t mmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pRpc); -} +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pRpc); } int32_t mmStartWorker(SMnodeMgmt *pMgmt) { SSingleWorkerCfg qCfg = { @@ -198,18 +176,16 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { return -1; } - if (tsMultiProcess) { - SSingleWorkerCfg mCfg = { - .min = 1, - .max = 1, - .name = "mnode-monitor", - .fp = (FItem)mmProcessQueue, - .param = pMgmt, - }; - if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start mnode mnode-monitor worker since %s", terrstr()); - return -1; - } + SSingleWorkerCfg mCfg = { + .min = 1, + .max = 1, + .name = "mnode-monitor", + .fp = (FItem)mmProcessQueue, + .param = pMgmt, + }; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start mnode mnode-monitor worker since %s", terrstr()); + return -1; } dDebug("mnode workers are initialized"); diff --git a/source/dnode/mgmt/mgmt_qnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_qnode/CMakeLists.txt index bf31b2afc3349593d3e54ef63c76c74762e87cc5..64f8a45ac428dff21cf241feda0669f87f9b8686 100644 --- a/source/dnode/mgmt/mgmt_qnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_qnode/CMakeLists.txt @@ -5,5 +5,5 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mgmt_qnode dnode_interface + mgmt_qnode node_util ) \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_qnode/inc/qmInt.h b/source/dnode/mgmt/mgmt_qnode/inc/qmInt.h index d52fbff683a5c5d43de21151d3e28b3e132e9620..8b48113dd31f3bdfd42887a75c386ced1cba85e9 100644 --- a/source/dnode/mgmt/mgmt_qnode/inc/qmInt.h +++ b/source/dnode/mgmt/mgmt_qnode/inc/qmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_QNODE_INT_H_ #define _TD_DND_QNODE_INT_H_ -#include "dmInt.h" +#include "dmUtil.h" #include "qnode.h" @@ -26,30 +26,31 @@ extern "C" { typedef struct SQnodeMgmt { SQnode *pQnode; - SDnode *pDnode; - SMgmtWrapper *pWrapper; + SMsgCb msgCb; const char *path; + const char *name; + int32_t dnodeId; SSingleWorker queryWorker; SSingleWorker fetchWorker; SSingleWorker monitorWorker; } SQnodeMgmt; // qmHandle.c -void qmInitMsgHandle(SMgmtWrapper *pWrapper); -int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); +SArray *qmGetMsgHandles(); +int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SNodeMsg *pMsg); +int32_t qmProcessDropReq(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SNodeMsg *pReq); // qmWorker.c -int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t qmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype); +int32_t qmPutRpcMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t qmPutRpcMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype); int32_t qmStartWorker(SQnodeMgmt *pMgmt); void qmStopWorker(SQnodeMgmt *pMgmt); -int32_t qmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t qmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t qmPutNodeMsgToFetchQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t qmPutNodeMsgToMonitorQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 11b91f056871eb5065d758c9c8d8a58b48828e19..c500176b15143d4e7fd744a99403fd5943ffa642 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -16,12 +16,12 @@ #define _DEFAULT_SOURCE #include "qmInt.h" -void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo) {} +static void qmGetMonitorInfo(SQnodeMgmt *pMgmt, SMonQmInfo *qmInfo) {} -int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { +int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SNodeMsg *pReq) { SMonQmInfo qmInfo = {0}; - qmGetMonitorInfo(pWrapper, &qmInfo); - dmGetMonitorSysInfo(&qmInfo.sys); + qmGetMonitorInfo(pMgmt, &qmInfo); + dmGetMonitorSystemInfo(&qmInfo.sys); monGetLogs(&qmInfo.log); int32_t rspLen = tSerializeSMonQmInfo(NULL, 0, &qmInfo); @@ -43,8 +43,7 @@ int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { return 0; } -int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SDnode *pDnode = pWrapper->pDnode; +int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDCreateQnodeReq createReq = {0}; @@ -53,14 +52,14 @@ int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } - if (createReq.dnodeId != pDnode->data.dnodeId) { + if (pInput->dnodeId != 0 && createReq.dnodeId != pInput->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; dError("failed to create qnode since %s", terrstr()); return -1; } bool deployed = true; - if (dmWriteFile(pWrapper, deployed) != 0) { + if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { dError("failed to write qnode file since %s", terrstr()); return -1; } @@ -68,8 +67,7 @@ int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SDnode *pDnode = pWrapper->pDnode; +int32_t qmProcessDropReq(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDDropQnodeReq dropReq = {0}; @@ -78,14 +76,14 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } - if (dropReq.dnodeId != pDnode->data.dnodeId) { + if (pMgmt->dnodeId != 0 && dropReq.dnodeId != pMgmt->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; dError("failed to drop qnode since %s", terrstr()); return -1; } bool deployed = false; - if (dmWriteFile(pWrapper, deployed) != 0) { + if (dmWriteFile(pMgmt->path, pMgmt->name, deployed) != 0) { dError("failed to write qnode file since %s", terrstr()); return -1; } @@ -93,18 +91,31 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -void qmInitMsgHandle(SMgmtWrapper *pWrapper) { - dmSetMsgHandle(pWrapper, TDMT_MON_QM_INFO, qmProcessMonitorMsg, DEFAULT_HANDLE); +SArray *qmGetMsgHandles() { + int32_t code = -1; + SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle)); + if (pArray == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_MON_QM_INFO, qmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; // Requests handled by VNODE - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, qmProcessFetchMsg, QNODE_HANDLE); - - dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QNODE_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QNODE_HANDLE); + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, qmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_RSP, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_VND_RES_READY, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASKS_STATUS, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + + code = 0; +_OVER: + if (code != 0) { + taosArrayDestroy(pArray); + return NULL; + } else { + return pArray; + } } diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmInt.c b/source/dnode/mgmt/mgmt_qnode/src/qmInt.c index d03f001a8d71c7a6fcfa329c6d667db7157ea461..5e86a307321f583849de0f6c412f22a4da05a5ad 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmInt.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmInt.c @@ -16,21 +16,13 @@ #define _DEFAULT_SOURCE #include "qmInt.h" -static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dmReadFile(pWrapper, required); } - -static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { - SMsgCb msgCb = pMgmt->pDnode->data.msgCb; - msgCb.pWrapper = pMgmt->pWrapper; - msgCb.queueFps[QUERY_QUEUE] = qmPutMsgToQueryQueue; - msgCb.queueFps[FETCH_QUEUE] = qmPutMsgToFetchQueue; - msgCb.qsizeFp = qmGetQueueSize; - pOption->msgCb = msgCb; +static int32_t qmRequire(const SMgmtInputOpt *pInput, bool *required) { + return dmReadFile(pInput->path, pInput->name, required); } -static void qmClose(SMgmtWrapper *pWrapper) { - SQnodeMgmt *pMgmt = pWrapper->pMgmt; - if (pMgmt == NULL) return; +static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { pOption->msgCb = pMgmt->msgCb; } +static void qmClose(SQnodeMgmt *pMgmt) { dInfo("qnode-mgmt start to cleanup"); if (pMgmt->pQnode != NULL) { qmStopWorker(pMgmt); @@ -38,12 +30,11 @@ static void qmClose(SMgmtWrapper *pWrapper) { pMgmt->pQnode = NULL; } - pWrapper->pMgmt = NULL; taosMemoryFree(pMgmt); dInfo("qnode-mgmt is cleaned up"); } -static int32_t qmOpen(SMgmtWrapper *pWrapper) { +static int32_t qmOpen(const SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dInfo("qnode-mgmt start to init"); SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt)); if (pMgmt == NULL) { @@ -51,41 +42,51 @@ static int32_t qmOpen(SMgmtWrapper *pWrapper) { return -1; } - pMgmt->path = pWrapper->path; - pMgmt->pDnode = pWrapper->pDnode; - pMgmt->pWrapper = pWrapper; - pWrapper->pMgmt = pMgmt; + pMgmt->path = pInput->path; + pMgmt->name = pInput->name; + pMgmt->dnodeId = pInput->dnodeId; + pMgmt->msgCb = pInput->msgCb; + pMgmt->msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qmPutRpcMsgToQueryQueue; + pMgmt->msgCb.queueFps[FETCH_QUEUE] = (PutToQueueFp)qmPutRpcMsgToFetchQueue; + pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)qmGetQueueSize; + pMgmt->msgCb.pMgmt = pMgmt; SQnodeOpt option = {0}; qmInitOption(pMgmt, &option); pMgmt->pQnode = qndOpen(&option); if (pMgmt->pQnode == NULL) { dError("failed to open qnode since %s", terrstr()); - qmClose(pWrapper); + qmClose(pMgmt); + return -1; + } + tmsgReportStartup("qnode-impl", "initialized"); + + if (udfcOpen() != 0) { + dError("qnode can not open udfc"); + qmClose(pMgmt); return -1; } - dmReportStartup(pWrapper->pDnode, "qnode-impl", "initialized"); if (qmStartWorker(pMgmt) != 0) { dError("failed to start qnode worker since %s", terrstr()); - qmClose(pWrapper); + qmClose(pMgmt); return -1; } - dmReportStartup(pWrapper->pDnode, "qnode-worker", "initialized"); + tmsgReportStartup("qnode-worker", "initialized"); + pOutput->pMgmt = pMgmt; dInfo("qnode-mgmt is initialized"); return 0; } -void qmSetMgmtFp(SMgmtWrapper *pWrapper) { - SMgmtFp mgmtFp = {0}; - mgmtFp.openFp = qmOpen; - mgmtFp.closeFp = qmClose; - mgmtFp.createFp = qmProcessCreateReq; - mgmtFp.dropFp = qmProcessDropReq; - mgmtFp.requiredFp = qmRequire; +SMgmtFunc qmGetMgmtFunc() { + SMgmtFunc mgmtFunc = {0}; + mgmtFunc.openFp = qmOpen; + mgmtFunc.closeFp = (NodeCloseFp)qmClose; + mgmtFunc.createFp = (NodeCreateFp)qmProcessCreateReq; + mgmtFunc.dropFp = (NodeDropFp)qmProcessDropReq; + mgmtFunc.requiredFp = qmRequire; + mgmtFunc.getHandlesFp = qmGetMsgHandles; - qmInitMsgHandle(pWrapper); - pWrapper->name = "qnode"; - pWrapper->fp = mgmtFp; + return mgmtFunc; } diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 965d35cb3e4546bdf7e4b48d4c4d9b13aef9c6ca..daac7f80bbc847fcb6e5d0113bb7637a2610a754 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -36,7 +36,7 @@ static void qmProcessMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { int32_t code = -1; if (pMsg->rpcMsg.msgType == TDMT_MON_QM_INFO) { - code = qmProcessGetMonQmInfoReq(pMgmt->pWrapper, pMsg); + code = qmProcessGetMonitorInfoReq(pMgmt, pMsg); } else { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } @@ -83,31 +83,26 @@ static void qmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { taosFreeQitem(pMsg); } -static void qmPutMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) { +static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) { dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); + return 0; } -int32_t qmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SQnodeMgmt *pMgmt = pWrapper->pMgmt; - qmPutMsgToWorker(&pMgmt->queryWorker, pMsg); - return 0; +int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); } -int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SQnodeMgmt *pMgmt = pWrapper->pMgmt; - qmPutMsgToWorker(&pMgmt->fetchWorker, pMsg); - return 0; +int32_t qmPutNodeMsgToFetchQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return qmPutNodeMsgToWorker(&pMgmt->fetchWorker, pMsg); } -int32_t qmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SQnodeMgmt *pMgmt = pWrapper->pMgmt; - qmPutMsgToWorker(&pMgmt->monitorWorker, pMsg); - return 0; +int32_t qmPutNodeMsgToMonitorQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return qmPutNodeMsgToWorker(&pMgmt->monitorWorker, pMsg); } static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { - SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg), RPC_QITEM); if (pMsg == NULL) { return -1; } @@ -118,26 +113,23 @@ static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SR return 0; } -int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - SQnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t qmPutRpcMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pRpc) { return qmPutRpcMsgToWorker(pMgmt, &pMgmt->queryWorker, pRpc); } -int32_t qmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - SQnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t qmPutRpcMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pRpc) { return qmPutRpcMsgToWorker(pMgmt, &pMgmt->fetchWorker, pRpc); } -int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { - int32_t size = -1; - SQnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { + int32_t size = -1; switch (qtype) { case QUERY_QUEUE: - size = taosQueueSize(pMgmt->queryWorker.queue); + size = taosQueueItemSize(pMgmt->queryWorker.queue); break; case FETCH_QUEUE: - size = taosQueueSize(pMgmt->fetchWorker.queue); + size = taosQueueItemSize(pMgmt->fetchWorker.queue); break; default: break; @@ -173,18 +165,16 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { return -1; } - if (tsMultiProcess) { - SSingleWorkerCfg mCfg = { - .min = 1, - .max = 1, - .name = "qnode-monitor", - .fp = (FItem)qmProcessMonitorQueue, - .param = pMgmt, - }; - if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start qnode-monitor worker since %s", terrstr()); - return -1; - } + SSingleWorkerCfg mCfg = { + .min = 1, + .max = 1, + .name = "qnode-monitor", + .fp = (FItem)qmProcessMonitorQueue, + .param = pMgmt, + }; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start qnode-monitor worker since %s", terrstr()); + return -1; } dDebug("qnode workers are initialized"); diff --git a/source/dnode/mgmt/mgmt_snode/CMakeLists.txt b/source/dnode/mgmt/mgmt_snode/CMakeLists.txt index b8a99c9d4df894c58854bef6a56b9b3daebcff6a..62dc41a0aec602f826ccdf5ea00f55691da62483 100644 --- a/source/dnode/mgmt/mgmt_snode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_snode/CMakeLists.txt @@ -5,5 +5,5 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mgmt_snode dnode_interface + mgmt_snode node_util ) \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_snode/inc/smInt.h b/source/dnode/mgmt/mgmt_snode/inc/smInt.h index 9eb48af733f5947b9318972719d7fae8c97a1119..a1ab9ba0776e206fd439e5389fb2b97d2d4899f1 100644 --- a/source/dnode/mgmt/mgmt_snode/inc/smInt.h +++ b/source/dnode/mgmt/mgmt_snode/inc/smInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_SNODE_INT_H_ #define _TD_DND_SNODE_INT_H_ -#include "dmInt.h" +#include "dmUtil.h" #include "snode.h" @@ -26,9 +26,10 @@ extern "C" { typedef struct SSnodeMgmt { SSnode *pSnode; - SDnode *pDnode; - SMgmtWrapper *pWrapper; + SMsgCb msgCb; const char *path; + const char *name; + int32_t dnodeId; SRWLatch latch; int8_t uniqueWorkerInUse; SArray *uniqueWorkers; // SArray @@ -37,19 +38,19 @@ typedef struct SSnodeMgmt { } SSnodeMgmt; // smHandle.c -void smInitMsgHandle(SMgmtWrapper *pWrapper); -int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); +SArray *smGetMsgHandles(); +int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SNodeMsg *pMsg); +int32_t smProcessDropReq(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smProcessGetMonitorInfoReq(SSnodeMgmt *pMgmt, SNodeMsg *pReq); // smWorker.c int32_t smStartWorker(SSnodeMgmt *pMgmt); void smStopWorker(SSnodeMgmt *pMgmt); -int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t smProcessExecMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smPutNodeMsgToExecQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t smPutNodeMsgToMonitorQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index defc5ab136b4703273f649e65385c6c3b291a92d..b0f0b73cbc2cdfda06fbe5853ebc92129c3552ec 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -16,12 +16,12 @@ #define _DEFAULT_SOURCE #include "smInt.h" -void smGetMonitorInfo(SMgmtWrapper *pWrapper, SMonSmInfo *smInfo) {} +static void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {} -int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { +int32_t smProcessGetMonitorInfoReq(SSnodeMgmt *pMgmt, SNodeMsg *pReq) { SMonSmInfo smInfo = {0}; - smGetMonitorInfo(pWrapper, &smInfo); - dmGetMonitorSysInfo(&smInfo.sys); + smGetMonitorInfo(pMgmt, &smInfo); + dmGetMonitorSystemInfo(&smInfo.sys); monGetLogs(&smInfo.log); int32_t rspLen = tSerializeSMonSmInfo(NULL, 0, &smInfo); @@ -43,8 +43,7 @@ int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { return 0; } -int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SDnode *pDnode = pWrapper->pDnode; +int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDCreateSnodeReq createReq = {0}; @@ -53,14 +52,14 @@ int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } - if (createReq.dnodeId != pDnode->data.dnodeId) { + if (pInput->dnodeId != 0 && createReq.dnodeId != pInput->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; dError("failed to create snode since %s", terrstr()); return -1; } bool deployed = true; - if (dmWriteFile(pWrapper, deployed) != 0) { + if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { dError("failed to write snode file since %s", terrstr()); return -1; } @@ -68,8 +67,7 @@ int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SDnode *pDnode = pWrapper->pDnode; +int32_t smProcessDropReq(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDDropSnodeReq dropReq = {0}; @@ -78,14 +76,14 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } - if (dropReq.dnodeId != pDnode->data.dnodeId) { + if (pMgmt->dnodeId != 0 && dropReq.dnodeId != pMgmt->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; dError("failed to drop snode since %s", terrstr()); return -1; } bool deployed = false; - if (dmWriteFile(pWrapper, deployed) != 0) { + if (dmWriteFile(pMgmt->path, pMgmt->name, deployed) != 0) { dError("failed to write snode file since %s", terrstr()); return -1; } @@ -93,10 +91,23 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -void smInitMsgHandle(SMgmtWrapper *pWrapper) { - dmSetMsgHandle(pWrapper, TDMT_MON_SM_INFO, smProcessMonitorMsg, DEFAULT_HANDLE); +SArray *smGetMsgHandles() { + int32_t code = -1; + SArray *pArray = taosArrayInit(4, sizeof(SMgmtHandle)); + if (pArray == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_MON_SM_INFO, smPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; // Requests handled by SNODE - dmSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE); + if (dmSetMgmtHandle(pArray, TDMT_SND_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SND_TASK_EXEC, smPutNodeMsgToExecQueue, 0) == NULL) goto _OVER; + + code = 0; +_OVER: + if (code != 0) { + taosArrayDestroy(pArray); + return NULL; + } else { + return pArray; + } } diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c index 26300a9fe3f22eda80354ab7188b32676f7a764c..5b9f2fc7f7055686f08022f69b845c8f3b34c426 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smInt.c +++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c @@ -17,34 +17,25 @@ #include "smInt.h" #include "libs/function/function.h" -static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dmReadFile(pWrapper, required); } - -static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { - SMsgCb msgCb = pMgmt->pDnode->data.msgCb; - msgCb.pWrapper = pMgmt->pWrapper; - pOption->msgCb = msgCb; +static int32_t smRequire(const SMgmtInputOpt *pInput, bool *required) { + return dmReadFile(pInput->path, pInput->name, required); } -static void smClose(SMgmtWrapper *pWrapper) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; - if (pMgmt == NULL) return; +static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { pOption->msgCb = pMgmt->msgCb; } +static void smClose(SSnodeMgmt *pMgmt) { dInfo("snode-mgmt start to cleanup"); - - udfcClose(); - if (pMgmt->pSnode != NULL) { smStopWorker(pMgmt); sndClose(pMgmt->pSnode); pMgmt->pSnode = NULL; } - pWrapper->pMgmt = NULL; taosMemoryFree(pMgmt); dInfo("snode-mgmt is cleaned up"); } -int32_t smOpen(SMgmtWrapper *pWrapper) { +int32_t smOpen(const SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dInfo("snode-mgmt start to init"); SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt)); if (pMgmt == NULL) { @@ -52,42 +43,47 @@ int32_t smOpen(SMgmtWrapper *pWrapper) { return -1; } - pMgmt->path = pWrapper->path; - pMgmt->pDnode = pWrapper->pDnode; - pMgmt->pWrapper = pWrapper; - pWrapper->pMgmt = pMgmt; + pMgmt->path = pInput->path; + pMgmt->name = pInput->name; + pMgmt->dnodeId = pInput->dnodeId; + pMgmt->msgCb = pInput->msgCb; + pMgmt->msgCb.pMgmt = pMgmt; SSnodeOpt option = {0}; smInitOption(pMgmt, &option); pMgmt->pSnode = sndOpen(pMgmt->path, &option); if (pMgmt->pSnode == NULL) { dError("failed to open snode since %s", terrstr()); + smClose(pMgmt); return -1; } - dmReportStartup(pWrapper->pDnode, "snode-impl", "initialized"); + tmsgReportStartup("snode-impl", "initialized"); if (smStartWorker(pMgmt) != 0) { dError("failed to start snode worker since %s", terrstr()); + smClose(pMgmt); return -1; } - dmReportStartup(pWrapper->pDnode, "snode-worker", "initialized"); + tmsgReportStartup("snode-worker", "initialized"); if (udfcOpen() != 0) { dError("failed to open udfc in snode"); + smClose(pMgmt); + return -1; } + pOutput->pMgmt = pMgmt; return 0; } -void smSetMgmtFp(SMgmtWrapper *pWrapper) { - SMgmtFp mgmtFp = {0}; - mgmtFp.openFp = smOpen; - mgmtFp.closeFp = smClose; - mgmtFp.createFp = smProcessCreateReq; - mgmtFp.dropFp = smProcessDropReq; - mgmtFp.requiredFp = smRequire; +SMgmtFunc smGetMgmtFunc() { + SMgmtFunc mgmtFunc = {0}; + mgmtFunc.openFp = smOpen; + mgmtFunc.closeFp = (NodeCloseFp)smClose; + mgmtFunc.createFp = (NodeCreateFp)smProcessCreateReq; + mgmtFunc.dropFp = (NodeDropFp)smProcessDropReq; + mgmtFunc.requiredFp = smRequire; + mgmtFunc.getHandlesFp = smGetMsgHandles; - smInitMsgHandle(pWrapper); - pWrapper->name = "snode"; - pWrapper->fp = mgmtFp; + return mgmtFunc; } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 2ae439bbd6a081bd21036bb3667d5aa0645df896..90e20f5fc5d0c6fbed35b940c5066b069ded84e5 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -36,7 +36,7 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { int32_t code = -1; if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) { - code = smProcessGetMonSmInfoReq(pMgmt->pWrapper, pMsg); + code = smProcessGetMonitorInfoReq(pMgmt, pMsg); } else { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } @@ -121,18 +121,16 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { return -1; } - if (tsMultiProcess) { - SSingleWorkerCfg mCfg = { - .min = 1, - .max = 1, - .name = "snode-monitor", - .fp = (FItem)smProcessMonitorQueue, - .param = pMgmt, - }; - if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start snode-monitor worker since %s", terrstr()); - return -1; - } + SSingleWorkerCfg mCfg = { + .min = 1, + .max = 1, + .name = "snode-monitor", + .fp = (FItem)smProcessMonitorQueue, + .param = pMgmt, + }; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start snode-monitor worker since %s", terrstr()); + return -1; } dDebug("snode workers are initialized"); @@ -163,8 +161,7 @@ static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { return 0; } -int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0); if (pWorker == NULL) { terrno = TSDB_CODE_INVALID_MSG; @@ -176,8 +173,7 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t smPutNodeMsgToMonitorQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -185,8 +181,7 @@ int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg); SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index); if (pWorker == NULL) { @@ -199,8 +194,7 @@ int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; +int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->sharedWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -208,11 +202,11 @@ int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t smProcessExecMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { +int32_t smPutNodeMsgToExecQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t workerType = smGetSWTypeFromMsg(&pMsg->rpcMsg); if (workerType == SND_WORKER_TYPE__SHARED) { - return smProcessSharedMsg(pWrapper, pMsg); + return smPutNodeMsgToSharedQueue(pMgmt, pMsg); } else { - return smProcessUniqueMsg(pWrapper, pMsg); + return smPutNodeMsgToUniqueQueue(pMgmt, pMsg); } } diff --git a/source/dnode/mgmt/mgmt_vnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_vnode/CMakeLists.txt index 55a76cf772ea66bdceded16f50e49001485dfbbd..15b822ad92e5367bcf15196f927e77d49a63f1bd 100644 --- a/source/dnode/mgmt/mgmt_vnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_vnode/CMakeLists.txt @@ -5,5 +5,5 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mgmt_vnode dnode_interface + mgmt_vnode node_util ) \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 5d8ec50e81b8df13285733abdbbde1ad49f9f6e5..ca43ef8c223a9675ee4b150655623e37fccf4957 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_VNODES_INT_H_ #define _TD_DND_VNODES_INT_H_ -#include "dmInt.h" +#include "dmUtil.h" #include "sync.h" #include "vnode.h" @@ -25,14 +25,11 @@ extern "C" { #endif -typedef struct SVnodesMgmt { - SHashObj *hash; - SRWLatch latch; - SVnodesStat state; +typedef struct SVnodeMgmt { + SMsgCb msgCb; const char *path; - SDnode *pDnode; - SMgmtWrapper *pWrapper; - STfs *pTfs; + const char *name; + int32_t dnodeId; SQWorkerPool queryPool; SQWorkerPool fetchPool; SWWorkerPool syncPool; @@ -40,7 +37,11 @@ typedef struct SVnodesMgmt { SWWorkerPool mergePool; SSingleWorker mgmtWorker; SSingleWorker monitorWorker; -} SVnodesMgmt; + SHashObj *hash; + SRWLatch latch; + SVnodesStat state; + STfs *pTfs; +} SVnodeMgmt; typedef struct { int32_t vgId; @@ -63,7 +64,6 @@ typedef struct { STaosQueue *pQueryQ; STaosQueue *pFetchQ; STaosQueue *pMergeQ; - SMgmtWrapper *pWrapper; } SVnodeObj; typedef struct { @@ -72,50 +72,49 @@ typedef struct { int32_t failed; int32_t threadIndex; TdThread thread; - SVnodesMgmt *pMgmt; + SVnodeMgmt *pMgmt; SWrapperCfg *pCfgs; } SVnodeThread; // vmInt.c -SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId); -void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); -int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); -void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); +SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId); +void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); +int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); +void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); // vmHandle.c -void vmInitMsgHandle(SMgmtWrapper *pWrapper); -int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); -int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); -int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); -int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); -void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo); +SArray *vmGetMsgHandles(); +int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessGetMonitorInfoReq(SVnodeMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessGetLoadsReq(SVnodeMgmt *pMgmt, SNodeMsg *pReq); // vmFile.c -int32_t vmGetVnodeListFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); -int32_t vmWriteVnodeListToFile(SVnodesMgmt *pMgmt); -SVnodeObj **vmGetVnodeListFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes); +int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); +int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt); +SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes); // vmWorker.c -int32_t vmStartWorker(SVnodesMgmt *pMgmt); -void vmStopWorker(SVnodesMgmt *pMgmt); -int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); -void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); - -int32_t vmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype); - -int32_t vmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t vmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t vmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t vmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrappert, SNodeMsg *pMsg); -int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t vmStartWorker(SVnodeMgmt *pMgmt); +void vmStopWorker(SVnodeMgmt *pMgmt); +int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); +void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); + +int32_t vmPutRpcMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutRpcMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutRpcMsgToApplyQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutRpcMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutRpcMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutRpcMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype); + +int32_t vmPutNodeMsgToWriteQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmPutNodeMsgToSyncQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmPutNodeMsgToQueryQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmPutNodeMsgToFetchQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmPutNodeMsgToMergeQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index ba4293eeb2ceb6a775281d624956d107643d7445..f6c7bb33e61595f886552baae062401c53aacf77 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" -SVnodeObj **vmGetVnodeListFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes) { +SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { taosRLockLatch(&pMgmt->latch); int32_t num = 0; @@ -44,7 +44,7 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes) { return pVnodes; } -int32_t vmGetVnodeListFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { +int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 1024 * 1024; @@ -139,9 +139,9 @@ _OVER: return code; } -int32_t vmWriteVnodeListToFile(SVnodesMgmt *pMgmt) { - char file[PATH_MAX]; - char realfile[PATH_MAX]; +int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { + char file[PATH_MAX] = {0}; + char realfile[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); snprintf(realfile, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 528f6a0ffe22d381b537c4c04eea985dc4aeae78..0814568b7324af50ad7847b4ab910e6d42752d92 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -16,9 +16,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" -void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; - +static void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad)); if (pInfo->pVloads == NULL) return; @@ -39,11 +37,9 @@ void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) { taosRUnLockLatch(&pMgmt->latch); } -void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *pInfo) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; - +static void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { SMonVloadInfo vloads = {0}; - vmGetVnodeLoads(pWrapper, &vloads); + vmGetVnodeLoads(pMgmt, &vloads); SArray *pVloads = vloads.pVloads; if (pVloads == NULL) return; @@ -86,10 +82,10 @@ void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *pInfo) { taosArrayDestroy(pVloads); } -int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { +int32_t vmProcessGetMonitorInfoReq(SVnodeMgmt *pMgmt, SNodeMsg *pReq) { SMonVmInfo vmInfo = {0}; - vmGetMonitorInfo(pWrapper, &vmInfo); - dmGetMonitorSysInfo(&vmInfo.sys); + vmGetMonitorInfo(pMgmt, &vmInfo); + dmGetMonitorSystemInfo(&vmInfo.sys); monGetLogs(&vmInfo.log); int32_t rspLen = tSerializeSMonVmInfo(NULL, 0, &vmInfo); @@ -111,9 +107,9 @@ int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { return 0; } -int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { +int32_t vmProcessGetLoadsReq(SVnodeMgmt *pMgmt, SNodeMsg *pReq) { SMonVloadInfo vloads = {0}; - vmGetVnodeLoads(pWrapper, &vloads); + vmGetVnodeLoads(pMgmt, &vloads); int32_t rspLen = tSerializeSMonVloadInfo(NULL, 0, &vloads); if (rspLen < 0) { @@ -170,14 +166,14 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { } } -static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { +static void vmGenerateWrapperCfg(SVnodeMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { pCfg->vgId = pCreate->vgId; pCfg->vgVersion = pCreate->vgVersion; pCfg->dropped = 0; snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCreate->vgId); } -int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { +int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SCreateVnodeReq createReq = {0}; int32_t code = -1; @@ -212,17 +208,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return -1; } - SMsgCb msgCb = pMgmt->pDnode->data.msgCb; - msgCb.pWrapper = pMgmt->pWrapper; - msgCb.queueFps[WRITE_QUEUE] = vmPutMsgToWriteQueue; - msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue; - msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; - msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; - msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; - msgCb.queueFps[MERGE_QUEUE] = vmPutMsgToMergeQueue; - msgCb.qsizeFp = vmGetQueueSize; - - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb); + SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); code = terrno; @@ -255,7 +241,7 @@ _OVER: return code; } -int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { +int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDropVnodeReq dropReq = {0}; if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { @@ -286,57 +272,71 @@ int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return 0; } -void vmInitMsgHandle(SMgmtWrapper *pWrapper) { - dmSetMsgHandle(pWrapper, TDMT_MON_VM_INFO, vmProcessMonitorMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MON_VM_LOAD, vmProcessMonitorMsg, DEFAULT_HANDLE); +SArray *vmGetMsgHandles() { + int32_t code = -1; + SArray *pArray = taosArrayInit(32, sizeof(SMgmtHandle)); + if (pArray == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_MON_VM_INFO, vmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; // Requests handled by VNODE - dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, vmProcessQueryMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, vmProcessQueryMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT_RSMA, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, vmProcessMergeMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, vmProcessSyncMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING, vmProcessSyncMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING_REPLY, vmProcessSyncMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST, vmProcessSyncMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, vmProcessSyncMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE, vmProcessSyncMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, vmProcessSyncMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES, vmProcessSyncMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, vmProcessSyncMsg, DEFAULT_HANDLE); + if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutNodeMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutNodeMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_TABLE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_UPDATE_TAG_VAL, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_META, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TABLES_META, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_CONSUME, vmPutNodeMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_QUERY, vmPutNodeMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_CONNECT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_DISCONNECT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + // if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_SET_CUR, vmPutNodeMsgToWriteQueue, 0)== NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_RES_READY, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASKS_STATUS, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_TASK, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_TABLE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_SMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT_RSMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_PIPE_EXEC, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_MERGE_EXEC, vmPutNodeMsgToMergeQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_WRITE_EXEC, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_TIMEOUT, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING_REPLY, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_CLIENT_REQUEST, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + + code = 0; + +_OVER: + if (code != 0) { + taosArrayDestroy(pArray); + return NULL; + } else { + return pArray; + } } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 9a0a524267a85c90872579ca07d93f35d0e31668..635467559e485582cc616139081c2d9579272ff2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" -SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { +SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; int32_t refCount = 0; @@ -36,7 +36,7 @@ SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { return pVnode; } -void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { +void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { if (pVnode == NULL) return; taosRLockLatch(&pMgmt->latch); @@ -45,7 +45,7 @@ void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } -int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { +int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { SVnodeObj *pVnode = taosMemoryCalloc(1, sizeof(SVnodeObj)); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -59,7 +59,6 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { pVnode->accessState = TSDB_VN_ALL_ACCCESS; pVnode->path = tstrdup(pCfg->path); pVnode->pImpl = pImpl; - pVnode->pWrapper = pMgmt->pWrapper; if (pVnode->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -78,7 +77,7 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { return code; } -void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { +void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { char path[TSDB_FILENAME_LEN] = {0}; taosWLockLatch(&pMgmt->latch); @@ -112,8 +111,7 @@ void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { static void *vmOpenVnodeInThread(void *param) { SVnodeThread *pThread = param; - SVnodesMgmt *pMgmt = pThread->pMgmt; - SDnode *pDnode = pMgmt->pDnode; + SVnodeMgmt *pMgmt = pThread->pMgmt; char path[TSDB_FILENAME_LEN]; dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); @@ -125,19 +123,10 @@ static void *vmOpenVnodeInThread(void *param) { char stepDesc[TSDB_STEP_DESC_LEN] = {0}; snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, pMgmt->state.openVnodes, pMgmt->state.totalVnodes); - dmReportStartup(pDnode, "vnode-open", stepDesc); - - SMsgCb msgCb = pMgmt->pDnode->data.msgCb; - msgCb.pWrapper = pMgmt->pWrapper; - msgCb.queueFps[WRITE_QUEUE] = vmPutMsgToWriteQueue; - msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue; - msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; - msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; - msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; - msgCb.queueFps[MERGE_QUEUE] = vmPutMsgToMergeQueue; - msgCb.qsizeFp = vmGetQueueSize; + tmsgReportStartup("vnode-open", stepDesc); + snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId); - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb); + SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; @@ -154,9 +143,7 @@ static void *vmOpenVnodeInThread(void *param) { return NULL; } -static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { - SDnode *pDnode = pMgmt->pDnode; - +static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pMgmt->hash == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -224,7 +211,7 @@ static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { } } -static void vmCloseVnodes(SVnodesMgmt *pMgmt) { +static void vmCloseVnodes(SVnodeMgmt *pMgmt) { dInfo("start to close all vnodes"); int32_t numOfVnodes = 0; @@ -246,40 +233,44 @@ static void vmCloseVnodes(SVnodesMgmt *pMgmt) { dInfo("total vnodes:%d are all closed", numOfVnodes); } -static void vmCleanup(SMgmtWrapper *pWrapper) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; - if (pMgmt == NULL) return; - +static void vmCleanup(SVnodeMgmt *pMgmt) { dInfo("vnode-mgmt start to cleanup"); vmCloseVnodes(pMgmt); vmStopWorker(pMgmt); vnodeCleanup(); tfsClose(pMgmt->pTfs); taosMemoryFree(pMgmt); - pWrapper->pMgmt = NULL; dInfo("vnode-mgmt is cleaned up"); } -static int32_t vmInit(SMgmtWrapper *pWrapper) { - SDnode *pDnode = pWrapper->pDnode; - SVnodesMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodesMgmt)); - int32_t code = -1; - +static int32_t vmInit(const SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dInfo("vnode-mgmt start to init"); + int32_t code = -1; + + SVnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodeMgmt)); if (pMgmt == NULL) goto _OVER; - pMgmt->path = pWrapper->path; - pMgmt->pDnode = pWrapper->pDnode; - pMgmt->pWrapper = pWrapper; + pMgmt->path = pInput->path; + pMgmt->name = pInput->name; + pMgmt->dnodeId = pInput->dnodeId; + pMgmt->msgCb = pInput->msgCb; + pMgmt->msgCb.queueFps[WRITE_QUEUE] = (PutToQueueFp)vmPutRpcMsgToWriteQueue; + pMgmt->msgCb.queueFps[SYNC_QUEUE] = (PutToQueueFp)vmPutRpcMsgToSyncQueue; + pMgmt->msgCb.queueFps[APPLY_QUEUE] = (PutToQueueFp)vmPutRpcMsgToApplyQueue; + pMgmt->msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)vmPutRpcMsgToQueryQueue; + pMgmt->msgCb.queueFps[FETCH_QUEUE] = (PutToQueueFp)vmPutRpcMsgToFetchQueue; + pMgmt->msgCb.queueFps[MERGE_QUEUE] = (PutToQueueFp)vmPutRpcMsgToMergeQueue; + pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize; + pMgmt->msgCb.pMgmt = pMgmt; taosInitRWLatch(&pMgmt->latch); SDiskCfg dCfg = {0}; - tstrncpy(dCfg.dir, pDnode->data.dataDir, TSDB_FILENAME_LEN); + tstrncpy(dCfg.dir, pInput->dataDir, TSDB_FILENAME_LEN); dCfg.level = 0; dCfg.primary = 1; - SDiskCfg *pDisks = pDnode->data.disks; - int32_t numOfDisks = pDnode->data.numOfDisks; + SDiskCfg *pDisks = pInput->disks; + int32_t numOfDisks = pInput->numOfDisks; if (numOfDisks <= 0 || pDisks == NULL) { pDisks = &dCfg; numOfDisks = 1; @@ -290,64 +281,64 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { dError("failed to init tfs since %s", terrstr()); goto _OVER; } - dmReportStartup(pDnode, "vnode-tfs", "initialized"); + tmsgReportStartup("vnode-tfs", "initialized"); if (walInit() != 0) { dError("failed to init wal since %s", terrstr()); goto _OVER; } - dmReportStartup(pDnode, "vnode-wal", "initialized"); + tmsgReportStartup("vnode-wal", "initialized"); if (syncInit() != 0) { dError("failed to open sync since %s", terrstr()); - return -1; + goto _OVER; } + tmsgReportStartup("vnode-sync", "initialized"); if (vnodeInit(tsNumOfCommitThreads) != 0) { dError("failed to init vnode since %s", terrstr()); goto _OVER; } - dmReportStartup(pDnode, "vnode-commit", "initialized"); + tmsgReportStartup("vnode-commit", "initialized"); if (vmStartWorker(pMgmt) != 0) { - dError("failed to init workers since %s", terrstr()) goto _OVER; + dError("failed to init workers since %s", terrstr()); + goto _OVER; } - dmReportStartup(pDnode, "vnode-worker", "initialized"); + tmsgReportStartup("vnode-worker", "initialized"); if (vmOpenVnodes(pMgmt) != 0) { dError("failed to open vnode since %s", terrstr()); - return -1; + goto _OVER; } - dmReportStartup(pDnode, "vnode-vnodes", "initialized"); + tmsgReportStartup("vnode-vnodes", "initialized"); if (udfcOpen() != 0) { dError("failed to open udfc in vnode"); + goto _OVER; } code = 0; _OVER: if (code == 0) { - pWrapper->pMgmt = pMgmt; + pOutput->pMgmt = pMgmt; dInfo("vnodes-mgmt is initialized"); } else { dError("failed to init vnodes-mgmt since %s", terrstr()); - vmCleanup(pWrapper); + vmCleanup(pMgmt); } - return 0; + return code; } -static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { - SDnode *pDnode = pWrapper->pDnode; - *required = pDnode->data.supportVnodes > 0; +static int32_t vmRequire(const SMgmtInputOpt *pInput, bool *required) { + *required = pInput->supportVnodes > 0; return 0; } -static int32_t vmStart(SMgmtWrapper *pWrapper) { +static int32_t vmStart(SVnodeMgmt *pMgmt) { dDebug("vnode-mgmt start to run"); - SVnodesMgmt *pMgmt = pWrapper->pMgmt; - taosRLockLatch(&pMgmt->latch); void *pIter = taosHashIterate(pMgmt->hash, NULL); @@ -364,20 +355,18 @@ static int32_t vmStart(SMgmtWrapper *pWrapper) { return 0; } -static void vmStop(SMgmtWrapper *pWrapper) { +static void vmStop(SVnodeMgmt *pMgmt) { // process inside the vnode } -void vmSetMgmtFp(SMgmtWrapper *pWrapper) { - SMgmtFp mgmtFp = {0}; - mgmtFp.openFp = vmInit; - mgmtFp.closeFp = vmCleanup; - mgmtFp.startFp = vmStart; - mgmtFp.stopFp = vmStop; - mgmtFp.requiredFp = vmRequire; - - vmInitMsgHandle(pWrapper); - pWrapper->name = "vnode"; - pWrapper->fp = mgmtFp; -} +SMgmtFunc vmGetMgmtFunc() { + SMgmtFunc mgmtFunc = {0}; + mgmtFunc.openFp = vmInit; + mgmtFunc.closeFp = (NodeCloseFp)vmCleanup; + mgmtFunc.startFp = (NodeStartFp)vmStart; + mgmtFunc.stopFp = (NodeStopFp)vmStop; + mgmtFunc.requiredFp = vmRequire; + mgmtFunc.getHandlesFp = vmGetMsgHandles; + return mgmtFunc; +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 2baa8b8942c996f818fc2ad56ce8798933b5b0d0..d6e99c0899b24e6b312960ba2fa86f2b070e5b9c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -32,7 +32,7 @@ static inline void vmSendRsp(SNodeMsg *pMsg, int32_t code) { } static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pInfo->ahandle; + SVnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; tmsg_t msgType = pMsg->rpcMsg.msgType; @@ -40,10 +40,10 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { switch (msgType) { case TDMT_MON_VM_INFO: - code = vmProcessGetMonVmInfoReq(pMgmt->pWrapper, pMsg); + code = vmProcessGetMonitorInfoReq(pMgmt, pMsg); break; case TDMT_MON_VM_LOAD: - code = vmProcessGetVnodeLoadsReq(pMgmt->pWrapper, pMsg); + code = vmProcessGetLoadsReq(pMgmt, pMsg); break; case TDMT_DND_CREATE_VNODE: code = vmProcessCreateVnodeReq(pMgmt, pMsg); @@ -240,7 +240,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } } -static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { +static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { SRpcMsg *pRpc = &pMsg->rpcMsg; SMsgHead *pHead = pRpc->pCont; int32_t code = 0; @@ -285,41 +285,34 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp return code; } -int32_t vmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; +int32_t vmPutNodeMsgToSyncQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); } -int32_t vmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; +int32_t vmPutNodeMsgToWriteQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, WRITE_QUEUE); } -int32_t vmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; +int32_t vmPutNodeMsgToQueryQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, QUERY_QUEUE); } -int32_t vmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; +int32_t vmPutNodeMsgToFetchQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); } -int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; +int32_t vmPutNodeMsgToMergeQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, MERGE_QUEUE); } -int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; +int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->mgmtWorker; dTrace("msg:%p, will be put into vnode-mgmt queue, worker:%s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } -int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; +int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SNodeMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, will be put into vnode-monitor queue, worker:%s", pMsg, pWorker->name); @@ -327,14 +320,13 @@ int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; - SMsgHead *pHead = pRpc->pCont; +static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType qtype) { + SMsgHead *pHead = pRpc->pCont; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) return -1; - SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg), RPC_QITEM); int32_t code = 0; if (pMsg != NULL) { @@ -377,62 +369,60 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT return code; } -int32_t vmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, WRITE_QUEUE); +int32_t vmPutRpcMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pMgmt, pRpc, WRITE_QUEUE); } -int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, SYNC_QUEUE); -} +int32_t vmPutRpcMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { return vmPutRpcMsgToQueue(pMgmt, pRpc, SYNC_QUEUE); } -int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, APPLY_QUEUE); +int32_t vmPutRpcMsgToApplyQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pMgmt, pRpc, APPLY_QUEUE); } -int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, QUERY_QUEUE); +int32_t vmPutRpcMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pMgmt, pRpc, QUERY_QUEUE); } -int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, FETCH_QUEUE); +int32_t vmPutRpcMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pMgmt, pRpc, FETCH_QUEUE); } -int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE); +int32_t vmPutRpcMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pMgmt, pRpc, MERGE_QUEUE); } -int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { +int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { int32_t size = -1; - SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId); + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode != NULL) { switch (qtype) { case WRITE_QUEUE: - size = taosQueueSize(pVnode->pWriteQ); + size = taosQueueItemSize(pVnode->pWriteQ); break; case SYNC_QUEUE: - size = taosQueueSize(pVnode->pSyncQ); + size = taosQueueItemSize(pVnode->pSyncQ); break; case APPLY_QUEUE: - size = taosQueueSize(pVnode->pApplyQ); + size = taosQueueItemSize(pVnode->pApplyQ); break; case QUERY_QUEUE: - size = taosQueueSize(pVnode->pQueryQ); + size = taosQueueItemSize(pVnode->pQueryQ); break; case FETCH_QUEUE: - size = taosQueueSize(pVnode->pFetchQ); + size = taosQueueItemSize(pVnode->pFetchQ); break; case MERGE_QUEUE: - size = taosQueueSize(pVnode->pMergeQ); + size = taosQueueItemSize(pVnode->pMergeQ); break; default: break; } } - vmReleaseVnode(pWrapper->pMgmt, pVnode); + vmReleaseVnode(pMgmt, pVnode); return size; } -int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { +int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); @@ -450,7 +440,7 @@ int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { return 0; } -void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { +void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); @@ -466,7 +456,7 @@ void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { dDebug("vgId:%d, vnode queue is freed", pVnode->vgId); } -int32_t vmStartWorker(SVnodesMgmt *pMgmt) { +int32_t vmStartWorker(SVnodeMgmt *pMgmt) { SQWorkerPool *pQPool = &pMgmt->queryPool; pQPool->name = "vnode-query"; pQPool->min = tsNumOfVnodeQueryThreads; @@ -506,25 +496,23 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { return -1; } - if (tsMultiProcess) { - SSingleWorkerCfg mCfg = { - .min = 1, - .max = 1, - .name = "vnode-monitor", - .fp = (FItem)vmProcessMgmtMonitorQueue, - .param = pMgmt, - }; - if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start mnode vnode-monitor worker since %s", terrstr()); - return -1; - } + SSingleWorkerCfg mCfg = { + .min = 1, + .max = 1, + .name = "vnode-monitor", + .fp = (FItem)vmProcessMgmtMonitorQueue, + .param = pMgmt, + }; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start mnode vnode-monitor worker since %s", terrstr()); + return -1; } dDebug("vnode workers are initialized"); return 0; } -void vmStopWorker(SVnodesMgmt *pMgmt) { +void vmStopWorker(SVnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->monitorWorker); tSingleWorkerCleanup(&pMgmt->mgmtWorker); tWWorkerCleanup(&pMgmt->writePool); diff --git a/source/dnode/mgmt/implement/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt similarity index 94% rename from source/dnode/mgmt/implement/CMakeLists.txt rename to source/dnode/mgmt/node_mgmt/CMakeLists.txt index fbe7530395e074bde7a780afa9914fcc31ec4d03..98027d80d4af858eef2a2108ef6d19b4b1851883 100644 --- a/source/dnode/mgmt/implement/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,9 +1,9 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_bnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode + dnode mgmt_bnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode ) target_include_directories( dnode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) \ No newline at end of file +) diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h new file mode 100644 index 0000000000000000000000000000000000000000..d717408fc63f6496cc762aa3d033f9487a7127ed --- /dev/null +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_IMP_H_ +#define _TD_DND_IMP_H_ + +// tobe deleted +#include "uv.h" + +#include "dmUtil.h" +#include "dmInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SMgmtWrapper { + SDnode *pDnode; + SMgmtFunc func; + void *pMgmt; + const char *name; + char *path; + int32_t refCount; + SRWLatch latch; + EDndNodeType nodeType; + bool deployed; + bool required; + EDndProcType procType; + int32_t procId; + SProcObj *procObj; + SShm procShm; + NodeMsgFp msgFps[TDMT_MAX]; +} SMgmtWrapper; + +typedef struct { + EDndNodeType defaultNtype; + bool needCheckVgId; +} SMsgHandle; + +typedef struct { + void *serverRpc; + void *clientRpc; + SMsgHandle msgHandles[TDMT_MAX]; +} SDnodeTrans; + +typedef struct { + char name[TSDB_STEP_NAME_LEN]; + char desc[TSDB_STEP_DESC_LEN]; +} SStartupInfo; + +typedef struct SUdfdData { + bool startCalled; + bool needCleanUp; + uv_loop_t loop; + uv_thread_t thread; + uv_barrier_t barrier; + uv_process_t process; + int spawnErr; + uv_pipe_t ctrlPipe; + uv_async_t stopAsync; + int32_t stopCalled; + int32_t dnodeId; +} SUdfdData; + +typedef struct SDnode { + EDndProcType ptype; + EDndNodeType ntype; + EDndEvent event; + EDndRunStatus status; + SStartupInfo startup; + SDnodeTrans trans; + SUdfdData udfdData; + TdThreadMutex mutex; + SRWLatch latch; + SEpSet mnodeEps; + TdFilePtr lockfile; + SMgmtInputOpt input; + SMgmtWrapper wrappers[NODE_END]; +} SDnode; + +// dmExec.c +int32_t dmOpenNode(SMgmtWrapper *pWrapper); +void dmCloseNode(SMgmtWrapper *pWrapper); + +// dmObj.c +SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType nType); +int32_t dmMarkWrapper(SMgmtWrapper *pWrapper); +void dmReleaseWrapper(SMgmtWrapper *pWrapper); + +void dmSetStatus(SDnode *pDnode, EDndRunStatus stype); +void dmSetEvent(SDnode *pDnode, EDndEvent event); +void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); +void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc); + +void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg); +void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg); +int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg); +int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg); + +// dmTransport.c +int32_t dmInitServer(SDnode *pDnode); +void dmCleanupServer(SDnode *pDnode); +int32_t dmInitClient(SDnode *pDnode); +void dmCleanupClient(SDnode *pDnode); +SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper); +SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper); +int32_t dmInitMsgHandle(SDnode *pDnode); + +// mgmt nodes +SMgmtFunc dmGetMgmtFunc(); +SMgmtFunc bmGetMgmtFunc(); +SMgmtFunc qmGetMgmtFunc(); +SMgmtFunc smGetMgmtFunc(); +SMgmtFunc vmGetMgmtFunc(); +SMgmtFunc mmGetMgmtFunc(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_IMP_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c new file mode 100644 index 0000000000000000000000000000000000000000..dbd861e6f7480e07c8c2f0f19848b14cfccb175f --- /dev/null +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -0,0 +1,358 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmMgmt.h" + +static bool dmIsNodeDeployedFp(SDnode *pDnode, EDndNodeType ntype) { return pDnode->wrappers[ntype].required; } + +static int32_t dmInitVars(SDnode *pDnode, const SDnodeOpt *pOption) { + pDnode->input.dnodeId = 0; + pDnode->input.clusterId = 0; + pDnode->input.localEp = strdup(pOption->localEp); + pDnode->input.localFqdn = strdup(pOption->localFqdn); + pDnode->input.firstEp = strdup(pOption->firstEp); + pDnode->input.secondEp = strdup(pOption->secondEp); + pDnode->input.serverPort = pOption->serverPort; + pDnode->input.supportVnodes = pOption->numOfSupportVnodes; + pDnode->input.numOfDisks = pOption->numOfDisks; + pDnode->input.disks = pOption->disks; + pDnode->input.dataDir = strdup(pOption->dataDir); + pDnode->input.pDnode = pDnode; + pDnode->input.processCreateNodeFp = dmProcessCreateNodeReq; + pDnode->input.processDropNodeFp = dmProcessDropNodeReq; + pDnode->input.isNodeDeployedFp = dmIsNodeDeployedFp; + + if (pDnode->input.dataDir == NULL || pDnode->input.localEp == NULL || pDnode->input.localFqdn == NULL || + pDnode->input.firstEp == NULL || pDnode->input.secondEp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pDnode->ntype = pOption->ntype; + if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_END) { + pDnode->lockfile = dmCheckRunning(pOption->dataDir); + if (pDnode->lockfile == NULL) { + return -1; + } + } + + taosThreadMutexInit(&pDnode->mutex, NULL); + return 0; +} + +static void dmClearVars(SDnode *pDnode) { + for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + taosMemoryFreeClear(pWrapper->path); + } + if (pDnode->lockfile != NULL) { + taosUnLockFile(pDnode->lockfile); + taosCloseFile(&pDnode->lockfile); + pDnode->lockfile = NULL; + } + + taosMemoryFreeClear(pDnode->input.localEp); + taosMemoryFreeClear(pDnode->input.localFqdn); + taosMemoryFreeClear(pDnode->input.firstEp); + taosMemoryFreeClear(pDnode->input.secondEp); + taosMemoryFreeClear(pDnode->input.dataDir); + + taosThreadMutexDestroy(&pDnode->mutex); + memset(&pDnode->mutex, 0, sizeof(pDnode->mutex)); + taosMemoryFree(pDnode); + dDebug("dnode memory is cleared, data:%p", pDnode); +} + +static bool dmRequireNode(SMgmtWrapper *pWrapper) { + SMgmtInputOpt *pInput = &pWrapper->pDnode->input; + pInput->name = pWrapper->name; + pInput->path = pWrapper->path; + + bool required = false; + int32_t code = (*pWrapper->func.requiredFp)(pInput, &required); + if (!required) { + dDebug("node:%s, does not require startup", pWrapper->name); + } + return required; +} + +SDnode *dmCreate(const SDnodeOpt *pOption) { + dInfo("start to create dnode"); + int32_t code = -1; + char path[PATH_MAX + 100] = {0}; + SDnode *pDnode = NULL; + + pDnode = taosMemoryCalloc(1, sizeof(SDnode)); + if (pDnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (dmInitVars(pDnode, pOption) != 0) { + goto _OVER; + } + + dmSetStatus(pDnode, DND_STAT_INIT); + pDnode->wrappers[DNODE].func = dmGetMgmtFunc(); + pDnode->wrappers[MNODE].func = mmGetMgmtFunc(); + pDnode->wrappers[VNODE].func = vmGetMgmtFunc(); + pDnode->wrappers[QNODE].func = qmGetMgmtFunc(); + pDnode->wrappers[SNODE].func = smGetMgmtFunc(); + pDnode->wrappers[BNODE].func = bmGetMgmtFunc(); + + for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + pWrapper->pDnode = pDnode; + pWrapper->name = dmNodeName(ntype); + pWrapper->procShm.id = -1; + pWrapper->nodeType = ntype; + pWrapper->procType = DND_PROC_SINGLE; + taosInitRWLatch(&pWrapper->latch); + + snprintf(path, sizeof(path), "%s%s%s", pOption->dataDir, TD_DIRSEP, pWrapper->name); + pWrapper->path = strdup(path); + if (pWrapper->path == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (ntype != DNODE && dmReadShmFile(pWrapper->path, pWrapper->name, pDnode->ntype, &pWrapper->procShm) != 0) { + dError("node:%s, failed to read shm file since %s", pWrapper->name, terrstr()); + goto _OVER; + } + + pWrapper->required = dmRequireNode(pWrapper); + } + + if (dmInitMsgHandle(pDnode) != 0) { + dError("failed to init msg handles since %s", terrstr()); + goto _OVER; + } + + if (dmInitClient(pDnode) != 0) { + goto _OVER; + } + + dInfo("dnode is created, data:%p", pDnode); + code = 0; + +_OVER: + if (code != 0 && pDnode != NULL) { + dmClearVars(pDnode); + pDnode = NULL; + dError("failed to create dnode since %s", terrstr()); + } + + return pDnode; +} + +void dmClose(SDnode *pDnode) { + if (pDnode == NULL) return; + + dmCleanupClient(pDnode); + dmCleanupServer(pDnode); + + dmClearVars(pDnode); + dInfo("dnode is closed, data:%p", pDnode); +} + +void dmSetStatus(SDnode *pDnode, EDndRunStatus status) { + if (pDnode->status != status) { + dDebug("dnode status set from %s to %s", dmStatStr(pDnode->status), dmStatStr(status)); + pDnode->status = status; + } +} + +void dmSetEvent(SDnode *pDnode, EDndEvent event) { + if (event == DND_EVENT_STOP) { + pDnode->event = event; + } +} + +SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + SMgmtWrapper *pRetWrapper = pWrapper; + + taosRLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); + dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount); + } else { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + pRetWrapper = NULL; + } + taosRUnLockLatch(&pWrapper->latch); + + return pRetWrapper; +} + +int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) { + int32_t code = 0; + + taosRLockLatch(&pWrapper->latch); + if (pWrapper->deployed || (pWrapper->procType == DND_PROC_PARENT && pWrapper->required)) { + int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); + dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount); + } else { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + code = -1; + } + taosRUnLockLatch(&pWrapper->latch); + + return code; +} + +void dmReleaseWrapper(SMgmtWrapper *pWrapper) { + if (pWrapper == NULL) return; + + taosRLockLatch(&pWrapper->latch); + int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); + taosRUnLockLatch(&pWrapper->latch); + dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount); +} + +void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupInfo *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + dInfo("step:%s, %s", pStartup->name, pStartup->desc); +} + +void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc) { + dmReportStartup(pWrapper->pDnode, pName, pDesc); +} + +static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus) { + SDnodeMgmt *pMgmt = pDnode->wrappers[DNODE].pMgmt; + pStatus->details[0] = 0; + + if (pDnode->status == DND_STAT_INIT) { + pStatus->statusCode = TSDB_SRV_STATUS_NETWORK_OK; + snprintf(pStatus->details, sizeof(pStatus->details), "%s: %s", pDnode->startup.name, pDnode->startup.desc); + } else if (pDnode->status == DND_STAT_STOPPED) { + pStatus->statusCode = TSDB_SRV_STATUS_EXTING; + } else { + pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK; + } +} + +void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("net test req is received"); + SRpcMsg rsp = {.handle = pReq->handle, .refId = pReq->refId, .ahandle = pReq->ahandle, .code = 0}; + rsp.pCont = rpcMallocCont(pReq->contLen); + if (rsp.pCont == NULL) { + rsp.code = TSDB_CODE_OUT_OF_MEMORY; + } else { + rsp.contLen = pReq->contLen; + } + rpcSendResponse(&rsp); + rpcFreeCont(pReq->pCont); +} + +void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("server startup status req is received"); + + SServerStatusRsp statusRsp = {0}; + dmGetServerStartupStatus(pDnode, &statusRsp); + + SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle, .refId = pReq->refId}; + int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); + if (rspLen < 0) { + rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp); + rspMsg.pCont = pRsp; + rspMsg.contLen = rspLen; + +_OVER: + rpcSendResponse(&rspMsg); + rpcFreeCont(pReq->pCont); +} + +int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); + if (pWrapper != NULL) { + dmReleaseWrapper(pWrapper); + terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + dError("failed to create node since %s", terrstr()); + return -1; + } + + taosThreadMutexLock(&pDnode->mutex); + pWrapper = &pDnode->wrappers[ntype]; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + SMgmtInputOpt *pInput = &pWrapper->pDnode->input; + pInput->name = pWrapper->name; + pInput->path = pWrapper->path; + pInput->msgCb = dmGetMsgcb(pWrapper); + + int32_t code = (*pWrapper->func.createFp)(pInput, pMsg); + if (code != 0) { + dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been created", pWrapper->name); + (void)dmOpenNode(pWrapper); + pWrapper->required = true; + pWrapper->deployed = true; + pWrapper->procType = pDnode->ptype; + } + + taosThreadMutexUnlock(&pDnode->mutex); + return code; +} + +int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + dError("failed to drop node since %s", terrstr()); + return -1; + } + + taosThreadMutexLock(&pDnode->mutex); + + int32_t code = (*pWrapper->func.dropFp)(pWrapper->pMgmt, pMsg); + if (code != 0) { + dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been dropped", pWrapper->name); + pWrapper->required = false; + pWrapper->deployed = false; + } + + dmReleaseWrapper(pWrapper); + + if (code == 0) { + dmCloseNode(pWrapper); + taosRemoveDir(pWrapper->path); + } + taosThreadMutexUnlock(&pDnode->mutex); + return code; +} \ No newline at end of file diff --git a/source/dnode/mgmt/implement/src/dmExec.c b/source/dnode/mgmt/node_mgmt/src/dmRun.c similarity index 73% rename from source/dnode/mgmt/implement/src/dmExec.c rename to source/dnode/mgmt/node_mgmt/src/dmRun.c index 6999cee0370059deca9af77cc9a8e036f85feeda..4d6290048aaac766b896f342b35ef715f4c64f4c 100644 --- a/source/dnode/mgmt/implement/src/dmExec.c +++ b/source/dnode/mgmt/node_mgmt/src/dmRun.c @@ -14,34 +14,25 @@ */ #define _DEFAULT_SOURCE -#include "dmImp.h" - -static bool dmRequireNode(SMgmtWrapper *pWrapper) { - bool required = false; - int32_t code = (*pWrapper->fp.requiredFp)(pWrapper, &required); - if (!required) { - dDebug("node:%s, does not require startup", pWrapper->name); - } - return required; -} +#include "dmMgmt.h" static int32_t dmInitParentProc(SMgmtWrapper *pWrapper) { int32_t shmsize = tsMnodeShmSize; - if (pWrapper->ntype == VNODE) { + if (pWrapper->nodeType == VNODE) { shmsize = tsVnodeShmSize; - } else if (pWrapper->ntype == QNODE) { + } else if (pWrapper->nodeType == QNODE) { shmsize = tsQnodeShmSize; - } else if (pWrapper->ntype == SNODE) { + } else if (pWrapper->nodeType == SNODE) { shmsize = tsSnodeShmSize; - } else if (pWrapper->ntype == MNODE) { + } else if (pWrapper->nodeType == MNODE) { shmsize = tsMnodeShmSize; - } else if (pWrapper->ntype == BNODE) { + } else if (pWrapper->nodeType == BNODE) { shmsize = tsBnodeShmSize; } else { return -1; } - if (taosCreateShm(&pWrapper->procShm, pWrapper->ntype, shmsize) != 0) { + if (taosCreateShm(&pWrapper->procShm, pWrapper->nodeType, shmsize) != 0) { terrno = TAOS_SYSTEM_ERROR(terrno); dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); return -1; @@ -86,7 +77,7 @@ static int32_t dmRunParentProc(SMgmtWrapper *pWrapper) { if (pWrapper->pDnode->ntype == NODE_END) { dInfo("node:%s, should be started manually in child process", pWrapper->name); } else { - if (dmNewNodeProc(pWrapper, pWrapper->ntype) != 0) { + if (dmNewNodeProc(pWrapper, pWrapper->nodeType) != 0) { return -1; } } @@ -123,8 +114,17 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) { return -1; } + SMgmtOutputOpt output = {0}; + SMgmtInputOpt *pInput = &pWrapper->pDnode->input; + pInput->name = pWrapper->name; + pInput->path = pWrapper->path; + pInput->msgCb = dmGetMsgcb(pWrapper); + if (pWrapper->nodeType == DNODE || pWrapper->procType == DND_PROC_CHILD) { + tmsgSetDefaultMsgCb(&pInput->msgCb); + } + if (pWrapper->procType == DND_PROC_SINGLE || pWrapper->procType == DND_PROC_CHILD) { - if ((*pWrapper->fp.openFp)(pWrapper) != 0) { + if ((*pWrapper->func.openFp)(pInput, &output) != 0) { dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); return -1; } @@ -136,27 +136,39 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) { pWrapper->deployed = true; } else { if (dmInitParentProc(pWrapper) != 0) return -1; - if (dmWriteShmFile(pWrapper) != 0) return -1; + if (dmWriteShmFile(pWrapper->path, pWrapper->name, &pWrapper->procShm) != 0) return -1; if (dmRunParentProc(pWrapper) != 0) return -1; } + if (output.dnodeId != 0) { + pInput->dnodeId = output.dnodeId; + } + if (output.pMgmt != NULL) { + pWrapper->pMgmt = output.pMgmt; + } + if (output.mnodeEps.numOfEps != 0) { + pWrapper->pDnode->mnodeEps = output.mnodeEps; + } + dmReportStartup(pWrapper->pDnode, pWrapper->name, "openned"); return 0; } int32_t dmStartNode(SMgmtWrapper *pWrapper) { + if (!pWrapper->required) return 0; + if (pWrapper->procType == DND_PROC_PARENT) { dInfo("node:%s, not start in parent process", pWrapper->name); } else if (pWrapper->procType == DND_PROC_CHILD) { dInfo("node:%s, start in child process", pWrapper->name); - if (pWrapper->ntype != DNODE) { - if (pWrapper->fp.startFp != NULL && (*pWrapper->fp.startFp)(pWrapper) != 0) { + if (pWrapper->nodeType != DNODE) { + if (pWrapper->func.startFp != NULL && (*pWrapper->func.startFp)(pWrapper->pMgmt) != 0) { dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; } } } else { - if (pWrapper->fp.startFp != NULL && (*pWrapper->fp.startFp)(pWrapper) != 0) { + if (pWrapper->func.startFp != NULL && (*pWrapper->func.startFp)(pWrapper->pMgmt) != 0) { dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; } @@ -167,8 +179,9 @@ int32_t dmStartNode(SMgmtWrapper *pWrapper) { } void dmStopNode(SMgmtWrapper *pWrapper) { - if (pWrapper->fp.stopFp != NULL) { - (*pWrapper->fp.stopFp)(pWrapper); + if (pWrapper->func.stopFp != NULL && pWrapper->pMgmt != NULL) { + (*pWrapper->func.stopFp)(pWrapper->pMgmt); + dDebug("node:%s, has been stopped", pWrapper->name); } } @@ -190,10 +203,11 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { } } - dmStopNode(pWrapper); - taosWLockLatch(&pWrapper->latch); - (*pWrapper->fp.closeFp)(pWrapper); + if (pWrapper->pMgmt != NULL) { + (*pWrapper->func.closeFp)(pWrapper->pMgmt); + pWrapper->pMgmt = NULL; + } taosWUnLockLatch(&pWrapper->latch); if (pWrapper->procObj) { @@ -205,49 +219,29 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { } static int32_t dmOpenNodes(SDnode *pDnode) { - if (pDnode->ptype == DND_PROC_CHILD) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; - pWrapper->required = dmRequireNode(pWrapper); - if (!pWrapper->required) { - dError("dnode:%s, failed to open since not required", pWrapper->name); - } - - pWrapper->procType = DND_PROC_CHILD; - if (dmInitClient(pDnode) != 0) { - return -1; - } - - pDnode->data.msgCb = dmGetMsgcb(pWrapper); - tmsgSetDefaultMsgCb(&pDnode->data.msgCb); - - if (dmOpenNode(pWrapper) != 0) { - dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); - return -1; - } - } else { - for (EDndNodeType n = DNODE; n < NODE_END; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - pWrapper->required = dmRequireNode(pWrapper); - if (!pWrapper->required) continue; - - if (pDnode->ptype == DND_PROC_PARENT && n != DNODE) { - pWrapper->procType = DND_PROC_PARENT; - } else { - pWrapper->procType = DND_PROC_SINGLE; + for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + if (!pWrapper->required) continue; + if (ntype == DNODE) { + pWrapper->procType = DND_PROC_SINGLE; + if (dmOpenNode(pWrapper) != 0) { + return -1; } - - if (n == DNODE) { - if (dmInitClient(pDnode) != 0) { + } else { + if (pDnode->ptype == DND_PROC_CHILD) { + if (pDnode->ntype == ntype) { + pWrapper->procType = DND_PROC_CHILD; + if (dmOpenNode(pWrapper) != 0) { + return -1; + } + } else { + pWrapper->required = false; + } + } else { + pWrapper->procType = pDnode->ptype; + if (dmOpenNode(pWrapper) != 0) { return -1; } - - pDnode->data.msgCb = dmGetMsgcb(pWrapper); - tmsgSetDefaultMsgCb(&pDnode->data.msgCb); - } - - if (dmOpenNode(pWrapper) != 0) { - dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); - return -1; } } } @@ -257,9 +251,9 @@ static int32_t dmOpenNodes(SDnode *pDnode) { } static int32_t dmStartNodes(SDnode *pDnode) { - for (EDndNodeType n = DNODE; n < NODE_END; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - if (!pWrapper->required) continue; + for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + if (ntype == DNODE && (pDnode->ptype == DND_PROC_CHILD || pDnode->ptype == DND_PROC_TEST)) continue; if (dmStartNode(pWrapper) != 0) { dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; @@ -313,16 +307,27 @@ static void dmWatchNodes(SDnode *pDnode) { } int32_t dmRun(SDnode *pDnode) { - if (!tsMultiProcess) { + if (tsMultiProcess == 0) { pDnode->ptype = DND_PROC_SINGLE; - dInfo("dnode run in single process"); + dInfo("dnode run in single process mode"); + } else if (tsMultiProcess == 2) { + pDnode->ptype = DND_PROC_TEST; + dInfo("dnode run in multi-process test mode"); } else if (pDnode->ntype == DNODE || pDnode->ntype == NODE_END) { pDnode->ptype = DND_PROC_PARENT; - dInfo("dnode run in parent process"); + dInfo("dnode run in parent process mode"); } else { pDnode->ptype = DND_PROC_CHILD; SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; - dInfo("%s run in child process", pWrapper->name); + dInfo("%s run in child process mode", pWrapper->name); + } + + if (pDnode->ptype != DND_PROC_CHILD) { + if (dmInitServer(pDnode) != 0) { + dError("failed to init transport since %s", terrstr()); + return -1; + } + dmReportStartup(pDnode, "dnode-transport", "initialized"); } if (dmOpenNodes(pDnode) != 0) { diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c similarity index 84% rename from source/dnode/mgmt/implement/src/dmTransport.c rename to source/dnode/mgmt/node_mgmt/src/dmTransport.c index a58999bf2d852e7af5bcb90f1865c26784221831..9f350409c848a8f0e6a9727412553d7f2d9b6f79 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -14,8 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "dmImp.h" - +#include "dmMgmt.h" #include "qworker.h" #define INTERNAL_USER "_dnd" @@ -23,21 +22,21 @@ #define INTERNAL_SECRET "_pwd" static void dmGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - taosRLockLatch(&pDnode->data.latch); - *pEpSet = pDnode->data.mnodeEps; - taosRUnLockLatch(&pDnode->data.latch); + taosRLockLatch(&pDnode->latch); + *pEpSet = pDnode->mnodeEps; + taosRUnLockLatch(&pDnode->latch); } static void dmSetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); - taosWLockLatch(&pDnode->data.latch); - pDnode->data.mnodeEps = *pEpSet; + taosWLockLatch(&pDnode->latch); + pDnode->mnodeEps = *pEpSet; for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } - taosWUnLockLatch(&pDnode->data.latch); + taosWUnLockLatch(&pDnode->latch); } static inline NodeMsgFp dmGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { @@ -64,7 +63,7 @@ static inline int32_t dmBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { if ((pRpc->msgType & 1u)) { assert(pRpc->refId != 0); } - // assert(pRpc->handle != NULL && pRpc->refId != 0 && pMsg->rpcMsg.refId != 0); + return 0; } @@ -76,20 +75,16 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe bool needRelease = false; bool isReq = msgType & 1U; - if (pEpSet && pEpSet->numOfEps > 0 && msgType == TDMT_MND_STATUS_RSP) { - dmSetMnodeEpSet(pWrapper->pDnode, pEpSet); - } - if (dmMarkWrapper(pWrapper) != 0) goto _OVER; - needRelease = true; + if ((msgFp = dmGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; - if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg), RPC_QITEM)) == NULL) goto _OVER; if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER; if (pWrapper->procType != DND_PROC_PARENT) { dTrace("msg:%p, created, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType), pRpc->handle, pMsg->user); - code = (*msgFp)(pWrapper, pMsg); + code = (*msgFp)(pWrapper->pMgmt, pMsg); } else { dTrace("msg:%p, created and put into child queue, type:%s handle:%p code:0x%04x user:%s contLen:%d", pMsg, TMSG_INFO(msgType), pRpc->handle, pMsg->rpcMsg.code & 0XFFFF, pMsg->user, pRpc->contLen); @@ -133,13 +128,17 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; - SMgmtWrapper *pWrapper = pHandle->pNdWrapper; + SMgmtWrapper *pWrapper = NULL; switch (msgType) { case TDMT_DND_SERVER_STATUS: - dTrace("server status req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); - dmProcessServerStatusReq(pDnode, pMsg); - return; + if (pDnode->status != DND_STAT_RUNNING) { + dTrace("server status req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dmProcessServerStartupStatus(pDnode, pMsg); + return; + } else { + break; + } case TDMT_DND_NET_TEST: dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); dmProcessNetTestReq(pDnode, pMsg); @@ -171,7 +170,7 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } - if (pWrapper == NULL) { + if (pHandle->defaultNtype == NODE_END) { dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); if (isReq) { SRpcMsg rspMsg = { @@ -182,13 +181,14 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } - if (pHandle->pMndWrapper != NULL || pHandle->pQndWrapper != NULL) { + pWrapper = &pDnode->wrappers[pHandle->defaultNtype]; + if (pHandle->needCheckVgId) { SMsgHead *pHead = pMsg->pCont; int32_t vgId = ntohl(pHead->vgId); if (vgId == QNODE_HANDLE) { - pWrapper = pHandle->pQndWrapper; + pWrapper = &pDnode->wrappers[QNODE]; } else if (vgId == MNODE_HANDLE) { - pWrapper = pHandle->pMndWrapper; + pWrapper = &pDnode->wrappers[MNODE]; } else { } } @@ -203,35 +203,24 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { int32_t dmInitMsgHandle(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; - for (EDndNodeType n = DNODE; n < NODE_END; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - - for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { - NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; - int8_t vgId = pWrapper->msgVgIds[msgIndex]; - if (msgFp == NULL) continue; + for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + SArray *pArray = (*pWrapper->func.getHandlesFp)(); + if (pArray == NULL) return -1; - SMsgHandle *pHandle = &pTrans->msgHandles[msgIndex]; - if (vgId == QNODE_HANDLE) { - if (pHandle->pQndWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pQndWrapper = pWrapper; - } else if (vgId == MNODE_HANDLE) { - if (pHandle->pMndWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pMndWrapper = pWrapper; - } else { - if (pHandle->pNdWrapper != NULL) { - dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); - return -1; - } - pHandle->pNdWrapper = pWrapper; + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SMgmtHandle *pMgmt = taosArrayGet(pArray, i); + SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; + if (pMgmt->needCheckVgId) { + pHandle->needCheckVgId = pMgmt->needCheckVgId; + } + if (!pMgmt->needCheckVgId) { + pHandle->defaultNtype = ntype; } + pWrapper->msgFps[TMSG_INDEX(pMgmt->msgType)] = pMgmt->msgFp; } + + taosArrayDestroy(pArray); } return 0; @@ -244,7 +233,7 @@ static void dmSendRpcRedirectRsp(SDnode *pDnode, const SRpcMsg *pReq) { dDebug("RPC %p, req is redirected, num:%d use:%d", pReq->handle, epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - if (strcmp(epSet.eps[i].fqdn, pDnode->data.localFqdn) == 0 && epSet.eps[i].port == pDnode->data.serverPort) { + if (strcmp(epSet.eps[i].fqdn, pDnode->input.localFqdn) == 0 && epSet.eps[i].port == pDnode->input.serverPort) { epSet.inUse = (i + 1) % epSet.numOfEps; } @@ -271,29 +260,32 @@ static inline void dmSendRpcRsp(SDnode *pDnode, const SRpcMsg *pRsp) { } } -void dmSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp) { - rpcSendRecv(pDnode->trans.clientRpc, pEpSet, pReq, pRsp); +static inline void dmSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp) { + if (pDnode->status != DND_STAT_RUNNING) { + pRsp->code = TSDB_CODE_NODE_OFFLINE; + rpcFreeCont(pReq->pCont); + pReq->pCont = NULL; + } else { + rpcSendRecv(pDnode->trans.clientRpc, pEpSet, pReq, pRsp); + } } -void dmSendToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) { +static inline void dmSendToMnodeRecv(SMgmtWrapper *pWrapper, SRpcMsg *pReq, SRpcMsg *pRsp) { SEpSet epSet = {0}; - dmGetMnodeEpSet(pDnode, &epSet); - rpcSendRecv(pDnode->trans.clientRpc, &epSet, pReq, pRsp); + dmGetMnodeEpSet(pWrapper->pDnode, &epSet); + dmSendRecv(pWrapper->pDnode, &epSet, pReq, pRsp); } static inline int32_t dmSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { SDnode *pDnode = pWrapper->pDnode; - if (pDnode->status != DND_STAT_RUNNING) { + if (pDnode->status != DND_STAT_RUNNING || pDnode->trans.clientRpc == NULL) { + rpcFreeCont(pReq->pCont); + pReq->pCont = NULL; terrno = TSDB_CODE_NODE_OFFLINE; dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); return -1; } - if (pDnode->trans.clientRpc == NULL) { - terrno = TSDB_CODE_NODE_OFFLINE; - return -1; - } - rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pReq, NULL); return 0; } @@ -326,17 +318,6 @@ static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp } } -#if 0 -static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { - ASSERT(pRsp->code == TSDB_CODE_RPC_REDIRECT); - if (pWrapper->procType != DND_PROC_CHILD) { - rpcSendRedirectRsp(pRsp->handle, pNewEpSet); - } else { - taosProcPutToParentQ(pWrapper->procObj, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_FUNC_RSP); - } -} -#endif - static inline void dmRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { if (pWrapper->procType != DND_PROC_CHILD) { rpcRegisterBrokenLinkArg(pMsg); @@ -361,7 +342,7 @@ static void dmConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - int32_t code = (*msgFp)(pWrapper, pMsg); + int32_t code = (*msgFp)(pWrapper->pMgmt, pMsg); if (code != 0) { dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); @@ -460,11 +441,7 @@ int32_t dmInitClient(SDnode *pDnode) { return -1; } - pDnode->data.msgCb = dmGetMsgcb(&pDnode->wrappers[DNODE]); - tmsgSetDefaultMsgCb(&pDnode->data.msgCb); - dDebug("dnode rpc client is initialized"); - return 0; } @@ -515,8 +492,10 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; SRpcMsg rpcRsp = {0}; + SEpSet epSet = {0}; dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); - dmSendToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); + dmGetMnodeEpSet(pDnode, &epSet); + dmSendRecv(pDnode, &epSet, &rpcMsg, &rpcRsp); if (rpcRsp.code != 0) { terrno = rpcRsp.code; @@ -541,8 +520,8 @@ int32_t dmInitServer(SDnode *pDnode) { SRpcInit rpcInit = {0}; - strncpy(rpcInit.localFqdn, pDnode->data.localFqdn, strlen(pDnode->data.localFqdn)); - rpcInit.localPort = pDnode->data.serverPort; + strncpy(rpcInit.localFqdn, pDnode->input.localFqdn, strlen(pDnode->input.localFqdn)); + rpcInit.localPort = pDnode->input.serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = tsNumOfRpcThreads; rpcInit.cfp = (RpcCfp)dmProcessMsg; @@ -573,14 +552,15 @@ void dmCleanupServer(SDnode *pDnode) { SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) { SMsgCb msgCb = { + .pWrapper = pWrapper, + .clientRpc = pWrapper->pDnode->trans.clientRpc, .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, + .sendMnodeRecvFp = dmSendToMnodeRecv, .sendRedirectRspFp = dmSendRedirectRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartupByWrapper, - .pWrapper = pWrapper, - .clientRpc = pWrapper->pDnode->trans.clientRpc, }; return msgCb; } diff --git a/source/dnode/mgmt/node_util/CMakeLists.txt b/source/dnode/mgmt/node_util/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..5d879fdbcf356f6cefcfdc8046696fa4010cd1b4 --- /dev/null +++ b/source/dnode/mgmt/node_util/CMakeLists.txt @@ -0,0 +1,10 @@ +aux_source_directory(src NODE_UTIL) +add_library(node_util STATIC ${NODE_UTIL}) +target_include_directories( + node_util + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mgmt" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + node_util cjson mnode vnode qnode snode bnode wal sync taos_static tfs monitor +) \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h new file mode 100644 index 0000000000000000000000000000000000000000..8d4ea88d420554973846050ceaf5f15263705c1c --- /dev/null +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DM_INT_H_ +#define _TD_DM_INT_H_ + +#include "cJSON.h" +#include "tcache.h" +#include "tcrc32c.h" +#include "tdatablock.h" +#include "tglobal.h" +#include "thash.h" +#include "tlockfree.h" +#include "tlog.h" +#include "tmsg.h" +#include "tmsgcb.h" +#include "tprocess.h" +#include "tqueue.h" +#include "trpc.h" +#include "tthread.h" +#include "ttime.h" +#include "tworker.h" + +#include "dnode.h" +#include "mnode.h" +#include "monitor.h" +#include "sync.h" +#include "wal.h" + +#include "libs/function/function.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} + +typedef enum { + DNODE = 0, + MNODE = 1, + VNODE = 2, + QNODE = 3, + SNODE = 4, + BNODE = 5, + NODE_END = 6, +} EDndNodeType; + +typedef enum { + DND_STAT_INIT, + DND_STAT_RUNNING, + DND_STAT_STOPPED, +} EDndRunStatus; + +typedef enum { + DND_ENV_INIT, + DND_ENV_READY, + DND_ENV_CLEANUP, +} EDndEnvStatus; + +typedef enum { + DND_PROC_SINGLE, + DND_PROC_CHILD, + DND_PROC_PARENT, + DND_PROC_TEST, +} EDndProcType; + +typedef int32_t (*ProcessCreateNodeFp)(struct SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg); +typedef int32_t (*ProcessDropNodeFp)(struct SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg); +typedef bool (*IsNodeDeployedFp)(struct SDnode *pDnode, EDndNodeType ntype); + +typedef struct { + const char *path; + const char *name; + SMsgCb msgCb; + int32_t dnodeId; + int64_t clusterId; + const char *localEp; + const char *firstEp; + const char *secondEp; + const char *localFqdn; + uint16_t serverPort; + int32_t supportVnodes; + int32_t numOfDisks; + SDiskCfg *disks; + const char *dataDir; + struct SDnode *pDnode; + ProcessCreateNodeFp processCreateNodeFp; + ProcessDropNodeFp processDropNodeFp; + IsNodeDeployedFp isNodeDeployedFp; +} SMgmtInputOpt; + +typedef struct { + int32_t dnodeId; + void *pMgmt; + SEpSet mnodeEps; +} SMgmtOutputOpt; + +typedef int32_t (*NodeMsgFp)(void *pMgmt, SNodeMsg *pMsg); +typedef int32_t (*NodeOpenFp)(const SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput); +typedef void (*NodeCloseFp)(void *pMgmt); +typedef int32_t (*NodeStartFp)(void *pMgmt); +typedef void (*NodeStopFp)(void *pMgmt); +typedef int32_t (*NodeCreateFp)(const SMgmtInputOpt *pInput, SNodeMsg *pMsg); +typedef int32_t (*NodeDropFp)(void *pMgmt, SNodeMsg *pMsg); +typedef int32_t (*NodeRequireFp)(const SMgmtInputOpt *pInput, bool *required); +typedef SArray *(*NodeGetHandlesFp)(); // array of SMgmtHandle + +typedef struct { + NodeOpenFp openFp; + NodeCloseFp closeFp; + NodeStartFp startFp; + NodeStopFp stopFp; + NodeCreateFp createFp; + NodeDropFp dropFp; + NodeRequireFp requiredFp; + NodeGetHandlesFp getHandlesFp; +} SMgmtFunc; + +typedef struct { + tmsg_t msgType; + bool needCheckVgId; + NodeMsgFp msgFp; +} SMgmtHandle; + +// dmUtil.c +const char *dmStatStr(EDndRunStatus stype); +const char *dmNodeLogName(EDndNodeType ntype); +const char *dmNodeProcName(EDndNodeType ntype); +const char *dmNodeName(EDndNodeType ntype); +const char *dmEventStr(EDndEvent etype); +const char *dmProcStr(EDndProcType ptype); +void *dmSetMgmtHandle(SArray *pArray, tmsg_t msgType, void *nodeMsgFp, bool needCheckVgId); +void dmGetMonitorSystemInfo(SMonSysInfo *pInfo); + +// dmFile.c +int32_t dmReadFile(const char *path, const char *name, bool *pDeployed); +int32_t dmWriteFile(const char *path, const char *name, bool deployed); +TdFilePtr dmCheckRunning(const char *dataDir); +int32_t dmReadShmFile(const char *path, const char *name, EDndNodeType runType, SShm *pShm); +int32_t dmWriteShmFile(const char *path, const char *name, const SShm *pShm); + +// common define +typedef struct { + int32_t dnodeId; + int64_t clusterId; + int64_t dnodeVer; + int64_t updateTime; + int64_t rebootTime; + int32_t unsyncedVgId; + ESyncState vndState; + ESyncState mndState; + bool dropped; + bool stopped; + SEpSet mnodeEps; + SArray *dnodeEps; + SHashObj *dnodeHash; + SRWLatch latch; + SMsgCb msgCb; + const char *localEp; + const char *localFqdn; + const char *firstEp; + const char *secondEp; + int32_t supportVnodes; + uint16_t serverPort; +} SDnodeData; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DM_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/interface/src/dmEnv.c b/source/dnode/mgmt/node_util/src/dmEnv.c similarity index 82% rename from source/dnode/mgmt/interface/src/dmEnv.c rename to source/dnode/mgmt/node_util/src/dmEnv.c index 2c836714cea3fd22c2c3fb647b74b00998a0ebbf..582df8055b8e6c389efe6bf9fba60d4070b03cfe 100644 --- a/source/dnode/mgmt/interface/src/dmEnv.c +++ b/source/dnode/mgmt/node_util/src/dmEnv.c @@ -14,16 +14,15 @@ */ #define _DEFAULT_SOURCE -#include "dmInt.h" -#include "wal.h" +#include "dmUtil.h" static int8_t once = DND_ENV_INIT; int32_t dmInit() { - dDebug("start to init dnode env"); + dInfo("start to init env"); if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { + dError("env is already initialized"); terrno = TSDB_CODE_REPEAT_INIT; - dError("failed to init dnode env since %s", terrstr()); return -1; } @@ -41,14 +40,14 @@ int32_t dmInit() { return -1; } - dInfo("dnode env is initialized"); + dInfo("env is initialized"); return 0; } void dmCleanup() { - dDebug("start to cleanup dnode env"); + dDebug("start to cleanup env"); if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { - dError("dnode env is already cleaned up"); + dError("env is already cleaned up"); return; } @@ -56,6 +55,7 @@ void dmCleanup() { syncCleanUp(); walCleanUp(); udfcClose(); + udfStopUdfd(); taosStopCacheRefreshWorker(); - dInfo("dnode env is cleaned up"); + dInfo("env is cleaned up"); } diff --git a/source/dnode/mgmt/interface/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c similarity index 72% rename from source/dnode/mgmt/interface/src/dmFile.c rename to source/dnode/mgmt/node_util/src/dmFile.c index 38acf169be47f51c96cbb0339f58b4b964c3d382..7ac6fc129d2bb591706d6ed722878359c4993515 100644 --- a/source/dnode/mgmt/interface/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -14,11 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "dmInt.h" +#include "dmUtil.h" #define MAXLEN 1024 -int32_t dmReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { +int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int64_t len = 0; char content[MAXLEN + 1] = {0}; @@ -26,10 +26,9 @@ int32_t dmReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { char file[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - // dDebug("file %s not exist", file); code = 0; goto _OVER; } @@ -64,7 +63,7 @@ _OVER: return code; } -int32_t dmWriteFile(SMgmtWrapper *pWrapper, bool deployed) { +int32_t dmWriteFile(const char *path, const char *name, bool deployed) { int32_t code = -1; int32_t len = 0; char content[MAXLEN + 1] = {0}; @@ -72,8 +71,8 @@ int32_t dmWriteFile(SMgmtWrapper *pWrapper, bool deployed) { char realfile[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); - snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); + snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); + snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -140,17 +139,16 @@ TdFilePtr dmCheckRunning(const char *dataDir) { return pFile; } -int32_t dmReadShmFile(SMgmtWrapper *pWrapper) { +int32_t dmReadShmFile(const char *path, const char *name, EDndNodeType runType, SShm *pShm) { int32_t code = -1; char content[MAXLEN + 1] = {0}; char file[PATH_MAX] = {0}; cJSON *root = NULL; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%sshmfile", pWrapper->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%sshmfile", path, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - // dDebug("node:%s, file %s not exist", pWrapper->name, file); code = 0; goto _OVER; } @@ -159,36 +157,36 @@ int32_t dmReadShmFile(SMgmtWrapper *pWrapper) { root = cJSON_Parse(content); if (root == NULL) { terrno = TSDB_CODE_INVALID_JSON_FORMAT; - dError("node:%s, failed to read %s since invalid json format", pWrapper->name, file); + dError("node:%s, failed to read %s since invalid json format", name, file); goto _OVER; } cJSON *shmid = cJSON_GetObjectItem(root, "shmid"); if (shmid && shmid->type == cJSON_Number) { - pWrapper->procShm.id = shmid->valueint; + pShm->id = shmid->valueint; } cJSON *shmsize = cJSON_GetObjectItem(root, "shmsize"); if (shmsize && shmsize->type == cJSON_Number) { - pWrapper->procShm.size = shmsize->valueint; + pShm->size = shmsize->valueint; } } - if (!tsMultiProcess || pWrapper->pDnode->ntype == DNODE || pWrapper->pDnode->ntype == NODE_END) { - if (pWrapper->procShm.id >= 0) { - dDebug("node:%s, shmid:%d, is closed, size:%d", pWrapper->name, pWrapper->procShm.id, pWrapper->procShm.size); - taosDropShm(&pWrapper->procShm); + if (!tsMultiProcess || runType == DNODE || runType == NODE_END) { + if (pShm->id >= 0) { + dDebug("node:%s, shmid:%d, is closed, size:%d", name, pShm->id, pShm->size); + taosDropShm(pShm); } } else { - if (taosAttachShm(&pWrapper->procShm) != 0) { + if (taosAttachShm(pShm) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("shmid:%d, failed to attach shm since %s", pWrapper->procShm.id, terrstr()); + dError("shmid:%d, failed to attach shm since %s", pShm->id, terrstr()); goto _OVER; } - dInfo("node:%s, shmid:%d is attached, size:%d", pWrapper->name, pWrapper->procShm.id, pWrapper->procShm.size); + dInfo("node:%s, shmid:%d is attached, size:%d", name, pShm->id, pShm->size); } - dDebug("node:%s, successed to load %s", pWrapper->name, file); + dDebug("node:%s, successed to load %s", name, file); code = 0; _OVER: @@ -198,7 +196,7 @@ _OVER: return code; } -int32_t dmWriteShmFile(SMgmtWrapper *pWrapper) { +int32_t dmWriteShmFile(const char *path, const char *name, const SShm *pShm) { int32_t code = -1; int32_t len = 0; char content[MAXLEN + 1] = {0}; @@ -206,30 +204,30 @@ int32_t dmWriteShmFile(SMgmtWrapper *pWrapper) { char realfile[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%sshmfile.bak", pWrapper->path, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%sshmfile", pWrapper->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%sshmfile.bak", path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%sshmfile", path, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("node:%s, failed to open file:%s since %s", pWrapper->name, file, terrstr()); + dError("node:%s, failed to open file:%s since %s", name, file, terrstr()); goto _OVER; } len += snprintf(content + len, MAXLEN - len, "{\n"); - len += snprintf(content + len, MAXLEN - len, " \"shmid\":%d,\n", pWrapper->procShm.id); - len += snprintf(content + len, MAXLEN - len, " \"shmsize\":%d\n", pWrapper->procShm.size); + len += snprintf(content + len, MAXLEN - len, " \"shmid\":%d,\n", pShm->id); + len += snprintf(content + len, MAXLEN - len, " \"shmsize\":%d\n", pShm->size); len += snprintf(content + len, MAXLEN - len, "}\n"); if (taosWriteFile(pFile, content, len) != len) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("node:%s, failed to write file:%s since %s", pWrapper->name, file, terrstr()); + dError("node:%s, failed to write file:%s since %s", name, file, terrstr()); goto _OVER; } if (taosFsyncFile(pFile) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("node:%s, failed to fsync file:%s since %s", pWrapper->name, file, terrstr()); + dError("node:%s, failed to fsync file:%s since %s", name, file, terrstr()); goto _OVER; } @@ -237,11 +235,11 @@ int32_t dmWriteShmFile(SMgmtWrapper *pWrapper) { if (taosRenameFile(file, realfile) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("node:%s, failed to rename %s to %s since %s", pWrapper->name, file, realfile, terrstr()); + dError("node:%s, failed to rename %s to %s since %s", name, file, realfile, terrstr()); return -1; } - dInfo("node:%s, successed to write %s", pWrapper->name, realfile); + dInfo("node:%s, successed to write %s", name, realfile); code = 0; _OVER: diff --git a/source/dnode/mgmt/node_util/src/dmUtil.c b/source/dnode/mgmt/node_util/src/dmUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..e913af203b76063e176da433e5773d400764e7d8 --- /dev/null +++ b/source/dnode/mgmt/node_util/src/dmUtil.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmUtil.h" + +const char *dmStatStr(EDndRunStatus stype) { + switch (stype) { + case DND_STAT_INIT: + return "init"; + case DND_STAT_RUNNING: + return "running"; + case DND_STAT_STOPPED: + return "stopped"; + default: + return "UNKNOWN"; + } +} + +const char *dmNodeLogName(EDndNodeType ntype) { + switch (ntype) { + case VNODE: + return "vnode"; + case QNODE: + return "qnode"; + case SNODE: + return "snode"; + case MNODE: + return "mnode"; + case BNODE: + return "bnode"; + default: + return "taosd"; + } +} + +const char *dmNodeProcName(EDndNodeType ntype) { + switch (ntype) { + case VNODE: + return "taosv"; + case QNODE: + return "taosq"; + case SNODE: + return "taoss"; + case MNODE: + return "taosm"; + case BNODE: + return "taosb"; + default: + return "taosd"; + } +} + +const char *dmNodeName(EDndNodeType ntype) { + switch (ntype) { + case VNODE: + return "vnode"; + case QNODE: + return "qnode"; + case SNODE: + return "snode"; + case MNODE: + return "mnode"; + case BNODE: + return "bnode"; + default: + return "dnode"; + } +} + +const char *dmEventStr(EDndEvent ev) { + switch (ev) { + case DND_EVENT_START: + return "start"; + case DND_EVENT_STOP: + return "stop"; + case DND_EVENT_CHILD: + return "child"; + default: + return "UNKNOWN"; + } +} + +const char *dmProcStr(EDndProcType etype) { + switch (etype) { + case DND_PROC_SINGLE: + return "start"; + case DND_PROC_CHILD: + return "stop"; + case DND_PROC_PARENT: + return "child"; + case DND_PROC_TEST: + return "test"; + default: + return "UNKNOWN"; + } +} + +void *dmSetMgmtHandle(SArray *pArray, tmsg_t msgType, void *nodeMsgFp, bool needCheckVgId) { + SMgmtHandle handle = { + .msgType = msgType, + .msgFp = (NodeMsgFp)nodeMsgFp, + .needCheckVgId = needCheckVgId, + }; + + return taosArrayPush(pArray, &handle); +} + +void dmGetMonitorSystemInfo(SMonSysInfo *pInfo) { + taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); + taosGetCpuCores(&pInfo->cpu_cores); + taosGetProcMemory(&pInfo->mem_engine); + taosGetSysMemory(&pInfo->mem_system); + pInfo->mem_total = tsTotalMemoryKB; + pInfo->disk_engine = 0; + pInfo->disk_used = tsDataSpace.size.used; + pInfo->disk_total = tsDataSpace.size.total; + taosGetCardInfoDelta(&pInfo->net_in, &pInfo->net_out); + taosGetProcIODelta(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); +} diff --git a/source/dnode/mgmt/test/CMakeLists.txt b/source/dnode/mgmt/test/CMakeLists.txt index e1656ceb34d222fb13ef524b087349756d46d6ff..6b1919bf1862b5eeca9047de4731dae306ca275a 100644 --- a/source/dnode/mgmt/test/CMakeLists.txt +++ b/source/dnode/mgmt/test/CMakeLists.txt @@ -3,7 +3,7 @@ if(${BUILD_TEST}) add_subdirectory(qnode) add_subdirectory(bnode) add_subdirectory(snode) - add_subdirectory(mnode) + #add_subdirectory(mnode) add_subdirectory(vnode) add_subdirectory(sut) endif(${BUILD_TEST}) diff --git a/source/dnode/mgmt/test/bnode/dbnode.cpp b/source/dnode/mgmt/test/bnode/dbnode.cpp index 9016bf49eac82b41ab5c8663d75c812f2be18cf2..4cc2f2386fa84cc52338628d13e2b2b051f162bc 100644 --- a/source/dnode/mgmt/test/bnode/dbnode.cpp +++ b/source/dnode/mgmt/test/bnode/dbnode.cpp @@ -84,6 +84,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { } TEST_F(DndTestBnode, 02_Drop_Bnode) { +#if 0 { SDDropBnodeReq dropReq = {0}; dropReq.dnodeId = 2; @@ -96,7 +97,7 @@ TEST_F(DndTestBnode, 02_Drop_Bnode) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); } - +#endif { SDDropBnodeReq dropReq = {0}; dropReq.dnodeId = 1; diff --git a/source/dnode/mgmt/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp index 8a0d97abb14e33054671f99bff3074effdc8af63..b610681b69d8d36f50aa112938a1b9f81c47993d 100644 --- a/source/dnode/mgmt/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -82,6 +82,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { } TEST_F(DndTestQnode, 02_Drop_Qnode) { +#if 0 { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; @@ -94,6 +95,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); } +#endif { SDDropQnodeReq dropReq = {0}; diff --git a/source/dnode/mgmt/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp index a744240a1aab7d781912d13af08ce042b9d13add..507531308580a6a112a62e6ab0d838c85f8b984d 100644 --- a/source/dnode/mgmt/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -82,6 +82,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { } TEST_F(DndTestSnode, 01_Drop_Snode) { +#if 0 { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; @@ -94,6 +95,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); } +#endif { SDDropSnodeReq dropReq = {0}; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index b96444bebcc7d9df5eff88d7dc878b9fa170be5f..75dda28a73ee048c2c0fc156727e7a65e708f756 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -87,13 +87,11 @@ typedef struct { typedef struct SMnode { int32_t selfId; int64_t clusterId; + TdThread thread; + bool stopped; int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; - tmr_h timer; - tmr_h transTimer; - tmr_h mqTimer; - tmr_h telemTimer; char *path; int64_t checkTime; SSdb *pSdb; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 690399f09995b484e1649f14ec25534d05ffa119..3c75e557e831efbbe188264ca87bda7badf06210 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -56,82 +56,72 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } -static void mndPullupTrans(void *param, void *tmrId) { - SMnode *pMnode = param; - if (mndIsMaster(pMnode)) { - int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - } - - taosTmrReset(mndPullupTrans, tsTransPullupInterval * 1000, pMnode, pMnode->timer, &pMnode->transTimer); +static void mndPullupTrans(SMnode *pMnode) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } -static void mndCalMqRebalance(void *param, void *tmrId) { - SMnode *pMnode = param; - if (mndIsMaster(pMnode)) { - int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_MQ_TIMER, - .pCont = pReq, - .contLen = contLen, - }; - tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); - } +static void mndCalMqRebalance(SMnode *pMnode) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); +} - taosTmrReset(mndCalMqRebalance, tsMqRebalanceInterval * 1000, pMnode, pMnode->timer, &pMnode->mqTimer); +static void mndPullupTelem(SMnode *pMnode) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } -static void mndPullupTelem(void *param, void *tmrId) { +static void *mndThreadFp(void *param) { SMnode *pMnode = param; - if (mndIsMaster(pMnode)) { - int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; - tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); - } + int64_t lastTime = 0; + setThreadName("mnode-timer"); - taosTmrReset(mndPullupTelem, tsTelemInterval * 1000, pMnode, pMnode->timer, &pMnode->telemTimer); -} + while (1) { + lastTime++; + taosMsleep(100); + if (pMnode->stopped) break; + if (!mndIsMaster(pMnode)) continue; -static int32_t mndInitTimer(SMnode *pMnode) { - pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); - if (pMnode->timer == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + if (lastTime % (tsTransPullupInterval * 10) == 0) { + mndPullupTrans(pMnode); + } - if (taosTmrReset(mndPullupTrans, tsTransPullupInterval * 1000, pMnode, pMnode->timer, &pMnode->transTimer)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + if (lastTime % (tsMqRebalanceInterval * 10) == 0) { + mndCalMqRebalance(pMnode); + } - if (taosTmrReset(mndCalMqRebalance, tsMqRebalanceInterval * 1000, pMnode, pMnode->timer, &pMnode->mqTimer)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + if (lastTime % (tsTelemInterval * 10) == 0) { + mndPullupTelem(pMnode); + } } - int32_t interval = tsTelemInterval < 10 ? tsTelemInterval : 10; - if (taosTmrReset(mndPullupTelem, interval * 1000, pMnode, pMnode->timer, &pMnode->telemTimer)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + +static int32_t mndInitTimer(SMnode *pMnode) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode) != 0) { + mError("failed to create timer thread since %s", strerror(errno)); return -1; } + taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("mnode-timer", "initialized"); return 0; } static void mndCleanupTimer(SMnode *pMnode) { - if (pMnode->timer != NULL) { - taosTmrStop(pMnode->transTimer); - pMnode->transTimer = NULL; - taosTmrStop(pMnode->mqTimer); - pMnode->mqTimer = NULL; - taosTmrStop(pMnode->telemTimer); - pMnode->telemTimer = NULL; - taosTmrCleanUp(pMnode->timer); - pMnode->timer = NULL; + pMnode->stopped = true; + if (taosCheckPthreadValid(pMnode->thread)) { + taosThreadJoin(pMnode->thread, NULL); } } diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index feeebad67484b950cb74d78c6c05396f83364532..b6e3c8f3b4b4e32d150f5989a2168358b9344c3e 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -5,7 +5,7 @@ add_subdirectory(bnode) add_subdirectory(db) add_subdirectory(dnode) add_subdirectory(func) -add_subdirectory(mnode) +#add_subdirectory(mnode) add_subdirectory(profile) add_subdirectory(qnode) add_subdirectory(sdb) diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 54b29f546cc888af80adaf1ec13c3b1a8c4d3efb..1259363f9468c2667536a17652010914dd4e2eac 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -26,10 +26,6 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { return NULL; } - if (udfcOpen() != 0) { - qError("qnode can not open udfc"); - } - if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { taosMemoryFreeClear(pQnode); return NULL; @@ -41,9 +37,6 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { void qndClose(SQnode *pQnode) { qWorkerDestroy((void **)&pQnode->pQuery); - - udfcClose(); - taosMemoryFree(pQnode); } diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index f1917d5fea02f126850495aed62496aa62f7f1a0..60b3a889ef0b7702c551ccb667432b7e41a00d43 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -96,8 +96,10 @@ typedef struct { #pragma pack(push, 1) typedef struct { tb_uid_t suid; - int16_t cid; - char data[]; + int32_t cid; + uint8_t isNull : 1; + uint8_t type : 7; + uint8_t data[]; // val + uid } STagIdxKey; #pragma pack(pop) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 8b3f555f4fa0a78bfeea2bc2a70d66c87787934b..07422e3193b6f77c252ea6442d14923d14a56242 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -227,8 +227,7 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { STagIdxKey *pTagIdxKey1 = (STagIdxKey *)pKey1; STagIdxKey *pTagIdxKey2 = (STagIdxKey *)pKey2; - int8_t *p1, *p2; - int8_t type; + tb_uid_t uid1, uid2; int c; // compare suid @@ -245,31 +244,34 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return -1; } - // compare value - p1 = pTagIdxKey1->data; - p2 = pTagIdxKey2->data; - ASSERT(p1[0] == p2[0]); - type = p1[0]; + ASSERT(pTagIdxKey1->type == pTagIdxKey2->type); - p1++; - p2++; - - c = doCompare(p1, p2, type, 0); - if (c) return c; - - if (IS_VAR_DATA_TYPE(type)) { - p1 = p1 + varDataTLen(p1); - p2 = p2 + varDataTLen(p2); - } else { - p1 = p1 + tDataTypes[type].bytes; - p2 = p2 + tDataTypes[type].bytes; + // check NULL, NULL is always the smallest + if (pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { + return -1; + } else if (!pTagIdxKey1->isNull && pTagIdxKey2->isNull) { + return 1; + } else if (!pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { + // all not NULL, compr tag vals + c = doCompare(pTagIdxKey1->data, pTagIdxKey2->data, pTagIdxKey1->type, 0); + if (c) return c; + + if (IS_VAR_DATA_TYPE(pTagIdxKey1->type)) { + uid1 = *(tb_uid_t *)(pTagIdxKey1->data + varDataTLen(pTagIdxKey1->data)); + uid2 = *(tb_uid_t *)(pTagIdxKey2->data + varDataTLen(pTagIdxKey2->data)); + } else { + uid1 = *(tb_uid_t *)(pTagIdxKey1->data + tDataTypes[pTagIdxKey1->type].bytes); + uid2 = *(tb_uid_t *)(pTagIdxKey2->data + tDataTypes[pTagIdxKey2->type].bytes); + } } - // compare suid - if (*(tb_uid_t *)p1 > *(tb_uid_t *)p2) { - return 1; - } else if (*(tb_uid_t *)p1 < *(tb_uid_t *)p2) { + // compare uid + if (uid1 < uid2) { return -1; + } else if (uid1 > uid2) { + return 1; + } else { + return 0; } return 0; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 20248f39fb26c33bcef9e336d46493f10a7f9336..d666bd22c1ead949a1a3e4f4e466051fb34d8530 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -22,7 +22,7 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; @@ -389,8 +389,73 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) { - // TODO +static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, + STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { + int32_t nTagData = 0; + + if (pTagData) { + if (IS_VAR_DATA_TYPE(type)) { + nTagData = varDataTLen(pTagData); + } else { + nTagData = tDataTypes[type].bytes; + } + } + *nTagIdxKey = sizeof(STagIdxKey) + nTagData + sizeof(tb_uid_t); + + *ppTagIdxKey = (STagIdxKey *)taosMemoryMalloc(*nTagIdxKey); + if (*ppTagIdxKey == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (*ppTagIdxKey)->suid = suid; + (*ppTagIdxKey)->cid = cid; + (*ppTagIdxKey)->isNull = (pTagData == NULL) ? 1 : 0; + (*ppTagIdxKey)->type = type; + if (nTagData) memcpy((*ppTagIdxKey)->data, pTagData, nTagData); + *(tb_uid_t *)((*ppTagIdxKey)->data + nTagData) = uid; + + return 0; +} + +static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { + if (pTagIdxKey) taosMemoryFree(pTagIdxKey); +} + +static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { + void *pData = NULL; + int nData = 0; + STbDbKey tbDbKey = {0}; + SMetaEntry stbEntry = {0}; + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; + const void *pTagData = NULL; // + SDecoder dc = {0}; + + // get super table + tdbDbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); + tbDbKey.uid = pCtbEntry->ctbEntry.suid; + tbDbKey.version = *(int64_t *)pData; + tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + + tDecoderInit(&dc, pData, nData); + metaDecodeEntry(&dc, &stbEntry); + + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + pTagData = tdGetKVRowValOfCol((const SKVRow)pCtbEntry->ctbEntry.pTags, pTagColumn->colId); + + // update tag index + if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, pTagColumn->type, pCtbEntry->uid, + &pTagIdxKey, &nTagIdxKey) < 0) { + return -1; + } + tdbDbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); + metaDestroyTagIdxKey(pTagIdxKey); + + tDecoderClear(&dc); + tdbFree(pData); + return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 32866d74696648fbeff7c6ea87d2ef489eb06ff5..a66ecc493d7cbef19370349568398d084dc5bc27 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -114,24 +114,42 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { int vnodeDecodeConfig(const SJson *pJson, void *pObj) { SVnodeCfg *pCfg = (SVnodeCfg *)pObj; - if (tjsonGetNumberValue(pJson, "vgId", pCfg->vgId) < 0) return -1; + int32_t code; + tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code); + if(code < 0) return -1; if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1; - if (tjsonGetNumberValue(pJson, "dbId", pCfg->dbId) < 0) return -1; - if (tjsonGetNumberValue(pJson, "szPage", pCfg->szPage) < 0) return -1; - if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1; - if (tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf) < 0) return -1; - if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap) < 0) return -1; - if (tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak) < 0) return -1; - if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; - if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; - if (tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; - if (tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1; - if (tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; - if (tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; - if (tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; - if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; - if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; - if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; + tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code); + if(code < 0) return -1; SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions"); int32_t nRetention = tjsonGetArraySize(pNodeRetentions); if (nRetention > TSDB_RETENTION_MAX) { @@ -140,24 +158,36 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { for (int32_t i = 0; i < nRetention; ++i) { SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i); ASSERT(pNodeRetention != NULL); - tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq); - tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit); - tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep); - tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit); + tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq, code); + tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit, code); + tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code); + tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code); } - if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; - if (tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; - if (tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; - if (tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; - - if (tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; - if (tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; + tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code); + if(code < 0) return -1; + + tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code); + if(code < 0) return -1; SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); int arraySize = tjsonGetArraySize(pNodeInfoArr); @@ -166,7 +196,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { for (int i = 0; i < arraySize; ++i) { SJson *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i); assert(pNodeInfo != NULL); - tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort); + tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort, code); tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn); } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 6d8bcb35c84a683b627b4a85dfc5e216c08b76e9..e7bee3342a39813e2825c9b9f29d201b6e68ca6e 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -310,8 +310,11 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) { static int vnodeDecodeState(const SJson *pJson, void *pObj) { SVState *pState = (SVState *)pObj; - if (tjsonGetNumberValue(pJson, "commit version", pState->committed) < 0) return -1; - if (tjsonGetNumberValue(pJson, "applied version", pState->applied) < 0) return -1; + int32_t code; + tjsonGetNumberValue(pJson, "commit version", pState->committed, code); + if(code < 0) return -1; + tjsonGetNumberValue(pJson, "applied version", pState->applied, code); + if(code < 0) return -1; return 0; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f485f85809318acdef81b7ec1e6f6c7fd34fd66d..64090d02831205b155cd477cf91d5d2ce2442956 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -841,6 +841,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable return TSDB_CODE_SUCCESS; } + + ctgDebug("Got subtable meta from cache, type:%d, dbFName:%s, tbName:%s, suid:%" PRIx64, tbMeta->tableType, dbFName, pTableName->tname, tbMeta->suid); CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); @@ -1655,6 +1657,11 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); if (orig) { origType = orig->tableType; + + if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; + } if (origType == TSDB_SUPER_TABLE) { if ((!isStb) || orig->suid != meta->suid) { @@ -1693,7 +1700,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_CACHE_STAT_ADD(tblNum, 1); } - ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64, dbFName, tbName, meta->tableType, meta->suid); ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { @@ -1701,12 +1708,6 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui return TSDB_CODE_SUCCESS; } - if (origType == TSDB_SUPER_TABLE && origSuid == meta->suid) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - return TSDB_CODE_SUCCESS; - } - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1721,7 +1722,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64 ",ma:%p", dbFName, tbName, meta->tableType, meta->suid, tbMeta); SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; strcpy(metaRent.dbFName, dbFName); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f79d3c8450cac0c17c09f44007cfa8cc44fbd638..dd0bcbff0e911980e0d8211707b8e2cf1d8bc71a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -564,7 +564,7 @@ typedef struct SStateWindowOperatorInfo { SAggSupporter aggSup; SGroupResInfo groupResInfo; SWindowRowsSup winSup; - int32_t colIndex; // start row index + SColumn stateCol; // start row index bool hasKey; SStateKeys stateKey; int32_t tsSlotId; // primary timestamp column slot id @@ -636,7 +636,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); -void doBuildResultDatablock(SExecTaskInfo *taskInfo, SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf); +void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); void finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, @@ -659,6 +659,7 @@ void cleanupAggSup(SAggSupporter* pAggSup); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); +SColumn extractColumnFromColumnNode(SColumnNode* pColNode); SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo); SSDataBlock* loadNextDataBlock(void* param); @@ -712,7 +713,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId, SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo); SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a217701471c07b1b2b86b25de7bc89ac9f400a45..fa9e27a5f810268f057a53d10b4d946dbd6825ea 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -88,9 +88,9 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery; - if (taosQueueSize(pDispatcher->pDataBlocks) > capacity) { + if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) { qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity, - taosQueueSize(pDispatcher->pDataBlocks)); + taosQueueItemSize(pDispatcher->pDataBlocks)); return false; } @@ -106,7 +106,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { taosThreadMutexLock(&pDispatcher->mutex); - int32_t blockNums = taosQueueSize(pDispatcher->pDataBlocks); + int32_t blockNums = taosQueueItemSize(pDispatcher->pDataBlocks); int32_t status = (0 == blockNums ? DS_BUF_EMPTY : (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); @@ -124,7 +124,7 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf)); + SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM); if (NULL == pBuf || !allocBuf(pDispatcher, pInput, pBuf)) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index df446def5ea3d665986a7575f7b723f3f96ea09c..556f094528003921589443b784814419335c1066 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -155,9 +155,8 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SExecTaskInfo *taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, int32_t orderType, int32_t* rowCellOffset, - SqlFunctionCtx* pCtx); +static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, + int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); @@ -898,6 +897,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; + colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; @@ -937,6 +937,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; + colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; @@ -2113,6 +2114,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } } +static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { if (pFilterNode == NULL) { return; @@ -2127,43 +2129,60 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; + // todo the keep seems never to be True?? bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); filterFreeInfo(filter); - SSDataBlock* px = createOneDataBlock(pBlock, false); - blockDataEnsureCapacity(px, pBlock->info.rows); + extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); + blockDataUpdateTsWindow(pBlock); +} - // todo extract method - int32_t numOfRow = 0; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); - if (keep) { - colDataAssign(pDst, pSrc, pBlock->info.rows); - numOfRow = pBlock->info.rows; - } else if (NULL != rowRes) { - numOfRow = 0; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { +void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { + if (keep) { + return; + } + + if (rowRes != NULL) { + SSDataBlock* px = createOneDataBlock(pBlock, false); + blockDataEnsureCapacity(px, pBlock->info.rows); + + int32_t totalRows = pBlock->info.rows; + + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); + + // For the reserved column, the value is not filled yet, so the whole column data may be NULL. + if (pSrc->pData == NULL) { + continue; + } + + int32_t numOfRows = 0; + for (int32_t j = 0; j < totalRows; ++j) { if (rowRes[j] == 0) { continue; } if (colDataIsNull_s(pSrc, j)) { - colDataAppendNULL(pDst, numOfRow); + colDataAppendNULL(pDst, numOfRows); } else { - colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false); + colDataAppend(pDst, numOfRows, colDataGetData(pSrc, j), false); } - numOfRow += 1; + numOfRows += 1; } - } else { - numOfRow = 0; - } - *pSrc = *pDst; - } + if (pBlock->info.rows == totalRows) { + pBlock->info.rows = numOfRows; + } else { + ASSERT(pBlock->info.rows == numOfRows); + } - pBlock->info.rows = numOfRow; - blockDataUpdateTsWindow(pBlock); + *pSrc = *pDst; + } + } else { + // do nothing + pBlock->info.rows = 0; + } } void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, @@ -2214,7 +2233,7 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p * @param result */ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { + int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -2248,13 +2267,12 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn pGroupResInfo->index += 1; - for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + for (int32_t j = 0; j < numOfExprs; ++j) { int32_t slotId = pExprInfo[j].base.resSchema.slotId; pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset); if (pCtx[j].fpSet.finalize) { - int32_t code = TSDB_CODE_SUCCESS; - code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); if (TAOS_FAILED(code)) { qError("%s build result data block error, code %s", GET_TASKID(taskInfo), tstrerror(code)); taskInfo->code = code; @@ -2286,10 +2304,13 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn return 0; } -void doBuildResultDatablock(SExecTaskInfo *taskInfo, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf) { +void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); + SExprInfo* pExprInfo = pOperator->pExpr; + int32_t numOfExprs = pOperator->numOfExprs; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; SSDataBlock* pBlock = pbInfo->pRes; SqlFunctionCtx* pCtx = pbInfo->pCtx; @@ -2300,7 +2321,7 @@ void doBuildResultDatablock(SExecTaskInfo *taskInfo, SOptrBasicInfo* pbInfo, SGr } int32_t orderType = TSDB_ORDER_ASC; - doCopyToSDataBlock(taskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx); + doCopyToSDataBlock(pTaskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx, numOfExprs); // add condition (pBlock->info.rows >= 1) just to runtime happy blockDataUpdateTsWindow(pBlock); @@ -3803,7 +3824,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { } blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pTaskInfo, pInfo, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -4963,7 +4984,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; - pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, pTaskInfo); + SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; + SColumn col = extractColumnFromColumnNode(pColNode); + pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); @@ -5028,6 +5051,17 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi return TSDB_CODE_SUCCESS; } +SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { + SColumn c = {0}; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; + c.precision = pColNode->node.resType.precision; + return c; +} + SArray* extractColumnInfo(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); @@ -5042,15 +5076,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - // todo extract method - SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.scale = pColNode->node.resType.scale; - c.precision = pColNode->node.resType.precision; - + SColumn c = extractColumnFromColumnNode(pColNode); taosArrayPush(pList, &c); } else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) { SValueNode* pValNode = (SValueNode*)pNode->pExpr; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index ac6f0cf8813def4c989202dce9b3bde14e5f1524..d8ccac8cea12bf210b962a58fa1274153ea8510e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -268,7 +268,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -317,7 +317,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); while(1) { - doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 332a116f762fb26f3892c884a83e9d8ae264015f..10dc482462f84a4dcdd7c53cb4bd164baa74f6af 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -806,10 +806,22 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } +static bool compareVal(const char* v, const SStateKeys* pKey) { + if (IS_VAR_DATA_TYPE(pKey->type)) { + if (varDataLen(v) != varDataLen(pKey->pData)) { + return false; + } else { + return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0; + } + } else { + return memcmp(pKey->pData, v, pKey->bytes) == 0; + } +} + static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); + SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId); int64_t gid = pBlock->info.groupId; bool masterScan = true; @@ -822,20 +834,28 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI SWindowRowsSup* pRowSup = &pInfo->winSup; pRowSup->numOfRows = 0; + struct SColumnDataAgg* pAgg = NULL; for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pBlock->pBlockAgg[pInfo->colIndex])) { + pAgg = (pBlock->pBlockAgg != NULL)? pBlock->pBlockAgg[pInfo->stateCol.slotId]: NULL; + if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) { continue; } char* val = colDataGetData(pStateColInfoData, j); if (!pInfo->hasKey) { - memcpy(pInfo->stateKey.pData, val, bytes); + // todo extract method + if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) { + varDataCopy(pInfo->stateKey.pData, val); + } else { + memcpy(pInfo->stateKey.pData, val, bytes); + } + pInfo->hasKey = true; doKeepNewWindowStartInfo(pRowSup, tsList, j); doKeepTuple(pRowSup, tsList[j]); - } else if (memcmp(pInfo->stateKey.pData, val, bytes) == 0) { + } else if (compareVal(val, &pInfo->stateKey)) { doKeepTuple(pRowSup, tsList[j]); if (j == 0 && pRowSup->startRowIndex != 0) { pRowSup->startRowIndex = 0; @@ -861,6 +881,13 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI // here we start a new session window doKeepNewWindowStartInfo(pRowSup, tsList, j); doKeepTuple(pRowSup, tsList[j]); + + // todo extract method + if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) { + varDataCopy(pInfo->stateKey.pData, val); + } else { + memcpy(pInfo->stateKey.pData, val, bytes); + } } } @@ -888,7 +915,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -921,7 +948,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -948,7 +975,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { } blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); - doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -1012,7 +1039,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pOperator->pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -1053,7 +1080,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pOperator->pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); // TODO: remove for stream /*ASSERT(pInfo->binfo.pRes->info.rows > 0);*/ @@ -1283,7 +1310,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pOperator->pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -1316,7 +1343,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pOperator->pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -1406,14 +1433,21 @@ _error: SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId, - SExecTaskInfo* pTaskInfo) { + SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo) { SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->colIndex = -1; + pInfo->stateCol = *pStateKeyCol; + pInfo->stateKey.type = pInfo->stateCol.type; + pInfo->stateKey.bytes = pInfo->stateCol.bytes; + pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes); + if (pInfo->stateKey.pData == NULL) { + goto _error; + } + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); @@ -1423,15 +1457,15 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf pInfo->twAggSup = *pTwAggSup; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - pInfo->tsSlotId = tsSlotId; - pOperator->name = "StateWindowOperator"; + pInfo->tsSlotId = tsSlotId; + pOperator->name = "StateWindowOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; pOperator->numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); diff --git a/source/libs/executor/test/index_executor_tests.cpp b/source/libs/executor/test/index_executor_tests.cpp index 5f1bff45a356273b95057cf97ea37d14c140b4e2..5b03da034e42504f65487fdf1bc4c075532b7e2c 100644 --- a/source/libs/executor/test/index_executor_tests.cpp +++ b/source/libs/executor/test/index_executor_tests.cpp @@ -200,11 +200,37 @@ TEST(testCase, index_filter) { doFilterTag(opNode, result); EXPECT_EQ(1, taosArrayGetSize(result)); + taosArrayDestroy(result); + nodesDestroyNode(res); + } + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); + SArray *result = taosArrayInit(4, sizeof(uint64_t)); + doFilterTag(opNode, result); + EXPECT_EQ(0, taosArrayGetSize(result)); + taosArrayDestroy(result); + nodesDestroyNode(res); + } + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + SArray *result = taosArrayInit(4, sizeof(uint64_t)); + doFilterTag(opNode, result); + EXPECT_EQ(0, taosArrayGetSize(result)); + taosArrayDestroy(result); nodesDestroyNode(res); } } +// add other greater/lower/equal/in compare func test + TEST(testCase, index_filter_varify) { { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index ea303eeadf745aa56f029771c3bfce42f4d0ea63..807234a1b15a7a5ed6040112e12eec6e7a6503bc 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -102,6 +102,11 @@ bool getMavgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool mavgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t mavgFunction(SqlFunctionCtx* pCtx); +bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool sampleFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t sampleFunction(SqlFunctionCtx* pCtx); +int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); #ifdef __cplusplus diff --git a/source/libs/function/inc/fnLog.h b/source/libs/function/inc/fnLog.h index f57294890794927e840a1a749f7e5adc5bbf9164..d85dd024331d0b2ec7374079cad9af344368a3c4 100644 --- a/source/libs/function/inc/fnLog.h +++ b/source/libs/function/inc/fnLog.h @@ -10,12 +10,12 @@ extern "C" { #endif -#define fnFatal(...) { if (fnDebugFlag & DEBUG_FATAL) { taosPrintLog("FN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define fnError(...) { if (fnDebugFlag & DEBUG_ERROR) { taosPrintLog("FN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define fnWarn(...) { if (fnDebugFlag & DEBUG_WARN) { taosPrintLog("FN WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define fnInfo(...) { if (fnDebugFlag & DEBUG_INFO) { taosPrintLog("FN ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define fnDebug(...) { if (fnDebugFlag & DEBUG_DEBUG) { taosPrintLog("FN ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} -#define fnTrace(...) { if (fnDebugFlag & DEBUG_TRACE) { taosPrintLog("FN ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} +#define fnFatal(...) { if (fnDebugFlag & DEBUG_FATAL) { taosPrintLog("FN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define fnError(...) { if (fnDebugFlag & DEBUG_ERROR) { taosPrintLog("FN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define fnWarn(...) { if (fnDebugFlag & DEBUG_WARN) { taosPrintLog("FN WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define fnInfo(...) { if (fnDebugFlag & DEBUG_INFO) { taosPrintLog("FN ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define fnDebug(...) { if (fnDebugFlag & DEBUG_DEBUG) { taosPrintLog("FN ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define fnTrace(...) { if (fnDebugFlag & DEBUG_TRACE) { taosPrintLog("FN ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index f5d1a5bf82b8e862892200fe7098df7d29a5996b..fc930083128c8bfe79f69e21f3c6737547ea67f1 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -360,6 +360,32 @@ static int32_t translateMavg(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return TSDB_CODE_SUCCESS; } +static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (2 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + if (QUERY_NODE_COLUMN != nodeType(pPara)) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "The input parameter of SAMPLE function can only be column"); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_INTEGER_TYPE(paraType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); + uint8_t colType = pCol->resType.type; + if (IS_VAR_DATA_TYPE(colType)) { + pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType}; + } else { + pFunc->node.resType = (SDataType){.bytes = tDataTypes[colType].bytes, .type = colType}; + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // todo return TSDB_CODE_SUCCESS; @@ -814,6 +840,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = mavgFunction, .finalizeFunc = NULL }, + { + .name = "sample", + .type = FUNCTION_TYPE_SAMPLE, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateSample, + .getEnvFunc = getSampleFuncEnv, + .initFunc = sampleFunctionSetup, + .processFunc = sampleFunction, + .finalizeFunc = NULL + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 2ec050d82d70a26e7a02b5a6571d496e6e43dc3c..af86eb4e90d2fab886b2c9c7f49d5df64d6176ca 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -23,6 +23,7 @@ #define HISTOGRAM_MAX_BINS_NUM 1000 #define MAVG_MAX_POINTS_NUM 1000 +#define SAMPLE_MAX_POINTS_NUM 1000 typedef struct SSumRes { union { @@ -150,6 +151,16 @@ typedef struct SMavgInfo { double points[]; } SMavgInfo; +typedef struct SSampleInfo { + int32_t samples; + int32_t totalPoints; + int32_t numSampled; + uint8_t colType; + int16_t colBytes; + char *data; + int64_t *timestamp; +} SSampleInfo; + #define SET_VAL(_info, numOfElem, res) \ do { \ if ((numOfElem) <= 0) { \ @@ -3034,3 +3045,99 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx) { return numOfElems; } + +bool getSampleFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); + SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); + int32_t numOfSamples = pVal->datum.i; + pEnv->calcMemSize = sizeof(SSampleInfo) + numOfSamples * (pCol->node.resType.bytes + sizeof(int64_t)); + return true; +} + +bool sampleFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + taosSeedRand(taosSafeRand()); + + SSampleInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + pInfo->samples = pCtx->param[1].param.i; + pInfo->totalPoints = 0; + pInfo->numSampled = 0; + pInfo->colType = pCtx->resDataInfo.type; + pInfo->colBytes = pCtx->resDataInfo.bytes; + if (pInfo->samples < 1 || pInfo->samples > SAMPLE_MAX_POINTS_NUM) { + return false; + } + pInfo->data = (char *)pInfo + sizeof(SSampleInfo); + pInfo->timestamp = (int64_t *)((char *)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes); + + return true; +} + +static void sampleAssignResult(SSampleInfo* pInfo, char *data, TSKEY ts, int32_t index) { + assignVal(pInfo->data + index * pInfo->colBytes, data, pInfo->colBytes, pInfo->colType); + *(pInfo->timestamp + index) = ts; +} + +static void doReservoirSample(SSampleInfo* pInfo, char *data, TSKEY ts, int32_t index) { + pInfo->totalPoints++; + if (pInfo->numSampled < pInfo->samples) { + sampleAssignResult(pInfo, data, ts, pInfo->numSampled); + pInfo->numSampled++; + } else { + int32_t j = taosRand() % (pInfo->totalPoints); + if (j < pInfo->samples) { + sampleAssignResult(pInfo, data, ts, j); + } + } +} + +int32_t sampleFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SSampleInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pTsOutput = pCtx->pTsOutput; + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + + int32_t startOffset = pCtx->offset; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + //colDataAppendNULL(pOutput, i); + continue; + } + + char* data = colDataGetData(pInputCol, i); + doReservoirSample(pInfo, data, tsList[i], i); + } + + for (int32_t i = 0; i < pInfo->numSampled; ++i) { + int32_t pos = startOffset + i; + colDataAppend(pOutput, pos, pInfo->data + i * pInfo->colBytes, false); + //TODO: handle ts output + } + + return pInfo->numSampled; +} + +//int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { +// SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); +// SSampleInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); +// int32_t slotId = pCtx->pExpr->base.resSchema.slotId; +// SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); +// +// //int32_t currentRow = pBlock->info.rows; +// pResInfo->numOfRes = pInfo->numSampled; +// +// for (int32_t i = 0; i < pInfo->numSampled; ++i) { +// colDataAppend(pCol, i, pInfo->data + i * pInfo->colBytes, false); +// //TODO: handle ts output +// } +// +// return pResInfo->numOfRes; +//} diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 8e96a2a0630ce86c05b3c87a83e9166c089ba1cf..3a388d1c07d5b6d797fef1b8151a1ef6d4443120 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -34,6 +34,9 @@ typedef struct SUdfdData { uv_thread_t thread; uv_barrier_t barrier; uv_process_t process; +#ifdef WINDOWS + HANDLE jobHandle; +#endif int spawnErr; uv_pipe_t ctrlPipe; uv_async_t stopAsync; @@ -104,6 +107,24 @@ static int32_t udfSpawnUdfd(SUdfdData* pData) { int err = uv_spawn(&pData->loop, &pData->process, &options); pData->process.data = (void*)pData; +#ifdef WINDOWS + // End udfd.exe by Job. + if (pData->jobHandle != NULL) CloseHandle(pData->jobHandle); + pData->jobHandle = CreateJobObject(NULL, NULL); + bool add_job_ok = AssignProcessToJobObject(pData->jobHandle, pData->process.process_handle); + if (!add_job_ok) { + fnError("Assign udfd to job failed."); + } else { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION limit_info; + memset(&limit_info, 0x0, sizeof(limit_info)); + limit_info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + bool set_auto_kill_ok = SetInformationJobObject(pData->jobHandle, JobObjectExtendedLimitInformation, &limit_info, sizeof(limit_info)); + if (!set_auto_kill_ok) { + fnError("Set job auto kill udfd failed."); + } + } +#endif + if (err != 0) { fnError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); } @@ -145,7 +166,7 @@ int32_t udfStartUdfd(int32_t startDnodeId) { } SUdfdData *pData = &udfdGlobal; if (pData->startCalled) { - fnInfo("dnode-mgmt start udfd already called"); + fnInfo("dnode start udfd already called"); return 0; } pData->startCalled = true; @@ -163,7 +184,7 @@ int32_t udfStartUdfd(int32_t startDnodeId) { uv_async_send(&pData->stopAsync); uv_thread_join(&pData->thread); pData->needCleanUp = false; - fnInfo("dnode-mgmt udfd cleaned up after spawn err"); + fnInfo("dnode udfd cleaned up after spawn err"); } else { pData->needCleanUp = true; } @@ -172,7 +193,7 @@ int32_t udfStartUdfd(int32_t startDnodeId) { int32_t udfStopUdfd() { SUdfdData *pData = &udfdGlobal; - fnInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d", + fnInfo("dnode to stop udfd. need cleanup: %d, spawn err: %d", pData->needCleanUp, pData->spawnErr); if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) { return 0; @@ -182,7 +203,10 @@ int32_t udfStopUdfd() { uv_barrier_destroy(&pData->barrier); uv_async_send(&pData->stopAsync); uv_thread_join(&pData->thread); - fnInfo("dnode-mgmt udfd cleaned up"); +#ifdef WINDOWS + if (pData->jobHandle != NULL) CloseHandle(pData->jobHandle); +#endif + fnInfo("dnode udfd cleaned up"); return 0; } @@ -286,28 +310,28 @@ enum { }; int64_t gUdfTaskSeqNum = 0; -typedef struct SUdfdProxy { +typedef struct SUdfcProxy { char udfdPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2]; - uv_barrier_t gUdfInitBarrier; + uv_barrier_t initBarrier; - uv_loop_t gUdfdLoop; - uv_thread_t gUdfLoopThread; - uv_async_t gUdfLoopTaskAync; + uv_loop_t uvLoop; + uv_thread_t loopThread; + uv_async_t loopTaskAync; - uv_async_t gUdfLoopStopAsync; + uv_async_t loopStopAsync; - uv_mutex_t gUdfTaskQueueMutex; - int8_t gUdfcState; - QUEUE gUdfTaskQueue; - QUEUE gUvProcTaskQueue; + uv_mutex_t taskQueueMutex; + int8_t udfcState; + QUEUE taskQueue; + QUEUE uvProcTaskQueue; int8_t initialized; -} SUdfdProxy; +} SUdfcProxy; -SUdfdProxy gUdfdProxy = {0}; +SUdfcProxy gUdfdProxy = {0}; typedef struct SClientUdfUvSession { - SUdfdProxy *udfc; + SUdfcProxy *udfc; int64_t severHandle; uv_pipe_t *udfUvPipe; @@ -317,7 +341,7 @@ typedef struct SClientUdfUvSession { } SClientUdfUvSession; typedef struct SClientUvTaskNode { - SUdfdProxy *udfc; + SUdfcProxy *udfc; int8_t type; int errCode; @@ -940,7 +964,7 @@ void udfcUvHandleError(SClientUvConn *conn) { uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); } -void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { +void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread); if (nread == 0) return; @@ -963,30 +987,32 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } -void onUdfClientWrite(uv_write_t *write, int status) { +void onUdfcPipetWrite(uv_write_t *write, int status) { SClientUvTaskNode *uvTask = write->data; uv_pipe_t *pipe = uvTask->pipe; + fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len); + SClientUvConn *conn = pipe->data; if (status == 0) { - SClientUvConn *conn = pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); } else { fnError("udfc client %p write error.", pipe); + udfcUvHandleError(conn); } - fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len); taosMemoryFree(write); taosMemoryFree(uvTask->reqBuf.base); } -void onUdfClientConnect(uv_connect_t *connect, int status) { +void onUdfcPipeConnect(uv_connect_t *connect, int status) { SClientUvTaskNode *uvTask = connect->data; - uvTask->errCode = status; if (status != 0) { - //TODO: LOG error + fnError("client connect error, task seq: %"PRId64", code: %s", uvTask->seqNum, uv_strerror(status)); } - uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); + uvTask->errCode = status; + + uv_read_start((uv_stream_t *)uvTask->pipe, udfcAllocateBuffer, onUdfcPipeRead); taosMemoryFree(connect); - uv_sem_post(&uvTask->taskSem); QUEUE_REMOVE(&uvTask->procTaskQueue); + uv_sem_post(&uvTask->taskSem); } int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { @@ -1031,11 +1057,11 @@ int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) { fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask); - SUdfdProxy *udfc = uvTask->udfc; - uv_mutex_lock(&udfc->gUdfTaskQueueMutex); - QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue); - uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); - uv_async_send(&udfc->gUdfLoopTaskAync); + SUdfcProxy *udfc = uvTask->udfc; + uv_mutex_lock(&udfc->taskQueueMutex); + QUEUE_INSERT_TAIL(&udfc->taskQueue, &uvTask->recvTaskQueue); + uv_mutex_unlock(&udfc->taskQueueMutex); + uv_async_send(&udfc->loopTaskAync); uv_sem_wait(&uvTask->taskSem); fnInfo("udfc uv task finished. task: %d, %p", uvTask->type, uvTask); @@ -1049,7 +1075,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); - uv_pipe_init(&uvTask->udfc->gUdfdLoop, pipe, 0); + uv_pipe_init(&uvTask->udfc->uvLoop, pipe, 0); uvTask->pipe = pipe; SClientUvConn *conn = taosMemoryCalloc(1, sizeof(SClientUvConn)); @@ -1064,14 +1090,17 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); connReq->data = uvTask; - uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfClientConnect); + uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfcPipeConnect); break; } case UV_TASK_REQ_RSP: { uv_pipe_t *pipe = uvTask->pipe; uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); write->data = uvTask; - uv_write(write, (uv_stream_t *) pipe, &uvTask->reqBuf, 1, onUdfClientWrite); + int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite); + if (err != 0) { + fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err)); + } break; } case UV_TASK_DISCONNECT: { @@ -1081,6 +1110,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { break; } default: { + fnError("udfc event loop unknown task type.") break; } } @@ -1089,46 +1119,48 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { } void udfClientAsyncCb(uv_async_t *async) { - SUdfdProxy *udfc = async->data; + SUdfcProxy *udfc = async->data; QUEUE wq; - uv_mutex_lock(&udfc->gUdfTaskQueueMutex); - QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); - uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); + uv_mutex_lock(&udfc->taskQueueMutex); + QUEUE_MOVE(&udfc->taskQueue, &wq); + uv_mutex_unlock(&udfc->taskQueueMutex); while (!QUEUE_EMPTY(&wq)) { QUEUE* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); - udfcStartUvTask(task); - QUEUE_INSERT_TAIL(&udfc->gUvProcTaskQueue, &task->procTaskQueue); + int32_t code = udfcStartUvTask(task); + if (code == 0) { + QUEUE_INSERT_TAIL(&udfc->uvProcTaskQueue, &task->procTaskQueue); + } } } -void cleanUpUvTasks(SUdfdProxy *udfc) { +void cleanUpUvTasks(SUdfcProxy *udfc) { fnDebug("clean up uv tasks") QUEUE wq; - uv_mutex_lock(&udfc->gUdfTaskQueueMutex); - QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); - uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); + uv_mutex_lock(&udfc->taskQueueMutex); + QUEUE_MOVE(&udfc->taskQueue, &wq); + uv_mutex_unlock(&udfc->taskQueueMutex); while (!QUEUE_EMPTY(&wq)) { QUEUE* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); - if (udfc->gUdfcState == UDFC_STATE_STOPPING) { + if (udfc->udfcState == UDFC_STATE_STOPPING) { task->errCode = TSDB_CODE_UDF_STOPPING; } uv_sem_post(&task->taskSem); } - while (!QUEUE_EMPTY(&udfc->gUvProcTaskQueue)) { - QUEUE* h = QUEUE_HEAD(&udfc->gUvProcTaskQueue); + while (!QUEUE_EMPTY(&udfc->uvProcTaskQueue)) { + QUEUE* h = QUEUE_HEAD(&udfc->uvProcTaskQueue); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue); - if (udfc->gUdfcState == UDFC_STATE_STOPPING) { + if (udfc->udfcState == UDFC_STATE_STOPPING) { task->errCode = TSDB_CODE_UDF_STOPPING; } uv_sem_post(&task->taskSem); @@ -1136,28 +1168,28 @@ void cleanUpUvTasks(SUdfdProxy *udfc) { } void udfStopAsyncCb(uv_async_t *async) { - SUdfdProxy *udfc = async->data; + SUdfcProxy *udfc = async->data; cleanUpUvTasks(udfc); - if (udfc->gUdfcState == UDFC_STATE_STOPPING) { - uv_stop(&udfc->gUdfdLoop); + if (udfc->udfcState == UDFC_STATE_STOPPING) { + uv_stop(&udfc->uvLoop); } } void constructUdfService(void *argsThread) { - SUdfdProxy *udfc = (SUdfdProxy*)argsThread; - uv_loop_init(&udfc->gUdfdLoop); - - uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopTaskAync, udfClientAsyncCb); - udfc->gUdfLoopTaskAync.data = udfc; - uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopStopAsync, udfStopAsyncCb); - udfc->gUdfLoopStopAsync.data = udfc; - uv_mutex_init(&udfc->gUdfTaskQueueMutex); - QUEUE_INIT(&udfc->gUdfTaskQueue); - QUEUE_INIT(&udfc->gUvProcTaskQueue); - uv_barrier_wait(&udfc->gUdfInitBarrier); + SUdfcProxy *udfc = (SUdfcProxy *)argsThread; + uv_loop_init(&udfc->uvLoop); + + uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfClientAsyncCb); + udfc->loopTaskAync.data = udfc; + uv_async_init(&udfc->uvLoop, &udfc->loopStopAsync, udfStopAsyncCb); + udfc->loopStopAsync.data = udfc; + uv_mutex_init(&udfc->taskQueueMutex); + QUEUE_INIT(&udfc->taskQueue); + QUEUE_INIT(&udfc->uvProcTaskQueue); + uv_barrier_wait(&udfc->initBarrier); //TODO return value of uv_run - uv_run(&udfc->gUdfdLoop, UV_RUN_DEFAULT); - uv_loop_close(&udfc->gUdfdLoop); + uv_run(&udfc->uvLoop, UV_RUN_DEFAULT); + uv_loop_close(&udfc->uvLoop); } int32_t udfcOpen() { @@ -1165,14 +1197,14 @@ int32_t udfcOpen() { if (old == 1) { return 0; } - SUdfdProxy *proxy = &gUdfdProxy; + SUdfcProxy *proxy = &gUdfdProxy; getUdfdPipeName(proxy->udfdPipeName, sizeof(proxy->udfdPipeName)); - proxy->gUdfcState = UDFC_STATE_STARTNG; - uv_barrier_init(&proxy->gUdfInitBarrier, 2); - uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy); - atomic_store_8(&proxy->gUdfcState, UDFC_STATE_READY); - proxy->gUdfcState = UDFC_STATE_READY; - uv_barrier_wait(&proxy->gUdfInitBarrier); + proxy->udfcState = UDFC_STATE_STARTNG; + uv_barrier_init(&proxy->initBarrier, 2); + uv_thread_create(&proxy->loopThread, constructUdfService, proxy); + atomic_store_8(&proxy->udfcState, UDFC_STATE_READY); + proxy->udfcState = UDFC_STATE_READY; + uv_barrier_wait(&proxy->initBarrier); fnInfo("udfc initialized") return 0; } @@ -1183,13 +1215,13 @@ int32_t udfcClose() { return 0; } - SUdfdProxy *udfc = &gUdfdProxy; - udfc->gUdfcState = UDFC_STATE_STOPPING; - uv_async_send(&udfc->gUdfLoopStopAsync); - uv_thread_join(&udfc->gUdfLoopThread); - uv_mutex_destroy(&udfc->gUdfTaskQueueMutex); - uv_barrier_destroy(&udfc->gUdfInitBarrier); - udfc->gUdfcState = UDFC_STATE_INITAL; + SUdfcProxy *udfc = &gUdfdProxy; + udfc->udfcState = UDFC_STATE_STOPPING; + uv_async_send(&udfc->loopStopAsync); + uv_thread_join(&udfc->loopThread); + uv_mutex_destroy(&udfc->taskQueueMutex); + uv_barrier_destroy(&udfc->initBarrier); + udfc->udfcState = UDFC_STATE_INITAL; fnInfo("udfc cleaned up"); return 0; } @@ -1212,7 +1244,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { fnInfo("udfc setup udf. udfName: %s", udfName); - if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) { + if (gUdfdProxy.udfcState != UDFC_STATE_READY) { return TSDB_CODE_UDF_INVALID_STATE; } SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask)); @@ -1460,7 +1492,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { SSDataBlock tempBlock = {0}; tempBlock.info.numOfCols = numOfCols; - tempBlock.info.rows = numOfRows; + tempBlock.info.rows = pInput->totalRows; tempBlock.info.uid = pInput->uid; bool hasVarCol = false; tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index ba39b09f56302964923cdfc5d073936bf46da3ee..6410af2a4b45b89b4aa894248507c176203430b6 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -44,7 +44,7 @@ int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInte case TSDB_DATA_TYPE_INT: { char* cell = udfColDataGetData(col, j); int32_t num = *(int32_t*)cell; - sumSquares += num * num; + sumSquares += (double)num * num; break; } case TSDB_DATA_TYPE_DOUBLE: { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e0c24ac3bd87ce6946ac289fba42b917d6585952..d56413f8401a3e262eb0a6982c0f94b12dad35df 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -27,10 +27,33 @@ #endif #define INDEX_NUM_OF_THREADS 4 -#define INDEX_QUEUE_SIZE 200 +#define INDEX_QUEUE_SIZE 200 void* indexQhandle = NULL; +#define INDEX_DATA_BOOL_NULL 0x02 +#define INDEX_DATA_TINYINT_NULL 0x80 +#define INDEX_DATA_SMALLINT_NULL 0x8000 +#define INDEX_DATA_INT_NULL 0x80000000L +#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L +#define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF +#define INDEX_DATA_BINARY_NULL 0xFF +#define INDEX_DATA_JSON_NULL 0xFFFFFFFF +#define INDEX_DATA_JSON_null 0xFFFFFFFE +#define INDEX_DATA_JSON_NOT_NULL 0x01 + +#define INDEX_DATA_UTINYINT_NULL 0xFF +#define INDEX_DATA_USMALLINT_NULL 0xFFFF +#define INDEX_DATA_UINT_NULL 0xFFFFFFFF +#define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define INDEX_DATA_NULL_STR "NULL" +#define INDEX_DATA_NULL_STR_L "null" + void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); @@ -67,12 +90,6 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { return -1; } -#ifdef USE_LUCENE - index_t* index = index_open(path); - sIdx->index = index; -#endif - -#ifdef USE_INVERTED_INDEX // sIdx->cache = (void*)indexCacheCreate(sIdx); sIdx->tindex = indexTFileCreate(path); if (sIdx->tindex == NULL) { @@ -85,7 +102,6 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadMutexInit(&sIdx->mtx, NULL); *index = sIdx; return 0; -#endif END: if (sIdx != NULL) { @@ -97,12 +113,6 @@ END: } void indexClose(SIndex* sIdx) { -#ifdef USE_LUCENE - index_close(sIdex->index); - sIdx->index = NULL; -#endif - -#ifdef USE_INVERTED_INDEX void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; @@ -114,31 +124,12 @@ void indexClose(SIndex* sIdx) { taosHashCleanup(sIdx->colObj); taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); -#endif taosMemoryFree(sIdx->path); taosMemoryFree(sIdx); return; } int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { -#ifdef USE_LUCENE - index_document_t* doc = index_document_create(); - - char buf[16] = {0}; - sprintf(buf, "%d", uid); - - for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm* p = taosArrayGetP(fVals, i); - index_document_add(doc, (const char*)(p->key), p->nKey, (const char*)(p->val), p->nVal, 1); - } - index_document_add(doc, NULL, 0, buf, strlen(buf), 0); - - index_put(index->index, doc); - index_document_destroy(doc); -#endif - -#ifdef USE_INVERTED_INDEX - // TODO(yihao): reduce the lock range taosThreadMutexLock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { @@ -170,12 +161,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { return ret; } } - -#endif return 0; } int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { -#ifdef USE_INVERTED_INDEX EIndexOperatorType opera = multiQuerys->opera; // relation of querys SArray* iRslts = taosArrayInit(4, POINTER_BYTES); @@ -188,35 +176,14 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result } indexMergeFinalResults(iRslts, opera, result); indexInterResultsDestroy(iRslts); - -#endif return 0; } -int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { -#ifdef USE_INVERTED_INDEX - -#endif +int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { return 1; } +int indexRebuild(SIndex* index, SIndexOpts* opts) { return 0; } - return 1; -} -int indexRebuild(SIndex* index, SIndexOpts* opts) { -#ifdef USE_INVERTED_INDEX -#endif - - return 0; -} - -SIndexOpts* indexOptsCreate() { -#ifdef USE_LUCENE -#endif - return NULL; -} -void indexOptsDestroy(SIndexOpts* opts) { -#ifdef USE_LUCENE -#endif - return; -} +SIndexOpts* indexOptsCreate() { return NULL; } +void indexOptsDestroy(SIndexOpts* opts) { return; } /* * @param: oper * diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 5f471dba65150b99daa98e280fdca87045f569b6..08d58da07fdc08b425e68f66f59e2433b2a1b76f 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -403,6 +403,19 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { EXPECT_EQ(1000, taosArrayGetSize(result)); indexMultiTermQueryDestroy(mq); } + { + std::string colName("other_column"); + std::string colVal("100"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 1000; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } { std::string colName("test1"); std::string colVal("10"); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index dba202a45f780ecf14fe7cc552f973b476866acb..6a80c293030b4cc8b2bb580a65c4b1af12711e4a 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -317,15 +317,16 @@ static int32_t tableComInfoToJson(const void* pObj, SJson* pJson) { static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) { STableComInfo* pNode = (STableComInfo*)pObj; - int32_t code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags); + int32_t code; + tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code);; if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision); + tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns); + tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize); + tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code);; } return code; @@ -356,12 +357,13 @@ static int32_t schemaToJson(const void* pObj, SJson* pJson) { static int32_t jsonToSchema(const SJson* pJson, void* pObj) { SSchema* pNode = (SSchema*)pObj; - int32_t code = tjsonGetNumberValue(pJson, jkSchemaType, pNode->type); + int32_t code; + tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code);; if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId); + tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes); + tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code);; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name); @@ -412,21 +414,22 @@ static int32_t tableMetaToJson(const void* pObj, SJson* pJson) { static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { STableMeta* pNode = (STableMeta*)pObj; - int32_t code = tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId); + int32_t code; + tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code);; if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType); + tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid); + tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid); + tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion); + tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion); + tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code);; } if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo); @@ -602,7 +605,7 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) { int32_t code = jsonToLogicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode); + tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code);; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs); @@ -878,7 +881,7 @@ static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkLogicSubplanRootNode, (SNode**)&pNode->pNode); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType); + tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType, code);; } int32_t objSize = 0; if (TSDB_CODE_SUCCESS == code) { @@ -1118,25 +1121,25 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanRatio, &pNode->ratio); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired); + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired, code);; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkTableScanPhysiPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableScanPhysiPlanInterval, pNode->interval); + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanInterval, pNode->interval, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableScanPhysiPlanOffset, pNode->offset); + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanOffset, pNode->offset, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSliding, pNode->sliding); + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSliding, pNode->sliding, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableScanPhysiPlanIntervalUnit, pNode->intervalUnit); + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanIntervalUnit, pNode->intervalUnit, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit); + tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit, code);; } return code; @@ -1182,7 +1185,7 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkSysTableScanPhysiPlanShowRewrite, &pNode->showRewrite); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId); + tjsonGetNumberValue(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId, code);; } return code; @@ -1266,7 +1269,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType); + tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions); @@ -1428,10 +1431,10 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkWindowPhysiPlanTsPk, (SNode**)&pNode->pTspk); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType); + tjsonGetNumberValue(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkWindowPhysiPlanWatermark, pNode->watermark); + tjsonGetNumberValue(pJson, jkWindowPhysiPlanWatermark, pNode->watermark, code);; } return code; @@ -1527,7 +1530,7 @@ static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode); + tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode, code);; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillPhysiPlanWStartTs, &pNode->pWStartTs); @@ -1566,7 +1569,7 @@ static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysiWindowNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap); + tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap, code);; } return code; @@ -1728,7 +1731,7 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { int32_t code = tjsonToObject(pJson, jkSubplanId, jsonToSubplanId, &pNode->id); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkSubplanType, pNode->subplanType); + tjsonGetNumberValue(pJson, jkSubplanType, pNode->subplanType, code);; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanMsgType, &pNode->msgType); @@ -1918,7 +1921,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType); + tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code);; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName); @@ -2172,7 +2175,7 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType); + tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code);; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft); @@ -2206,7 +2209,7 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType); + tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code);; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList); @@ -2412,10 +2415,10 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) { int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order); + tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code);; } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder); + tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code);; } return code; @@ -2521,7 +2524,8 @@ static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { SFillNode* pNode = (SFillNode*)pObj; - int32_t code = tjsonGetNumberValue(pJson, jkFillMode, pNode->mode); + int32_t code; + tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code);; if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); } @@ -3117,7 +3121,8 @@ static int32_t nodeToJson(const void* pObj, SJson* pJson) { static int32_t jsonToNode(const SJson* pJson, void* pObj) { SNode* pNode = (SNode*)pObj; - int32_t code = tjsonGetNumberValue(pJson, jkNodeType, pNode->type); + int32_t code; + tjsonGetNumberValue(pJson, jkNodeType, pNode->type, code);; if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 228cb5a44bf722ed2875b7e482e4bd97c9b0c03f..27383c0a516b48be03958c49c55f78f5beb3821d 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -758,7 +758,7 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi int32_t output = 0; if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); return buildSyntaxErrMsg(pMsgBuf, buf, value); } @@ -1227,16 +1227,20 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } - (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); - if (NULL == (*pQuery)->pTableList) { - return TSDB_CODE_OUT_OF_MEMORY; - } + (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; (*pQuery)->haveResultSet = false; (*pQuery)->msgType = TDMT_VND_SUBMIT; (*pQuery)->pRoot = (SNode*)context.pOutput; } + if (NULL == (*pQuery)->pTableList) { + (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); + if (NULL == (*pQuery)->pTableList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + context.pOutput->payloadType = PAYLOAD_TYPE_KV; int32_t code = skipInsertInto(&context); @@ -1668,7 +1672,11 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 param.schema = pTagSchema; SSmlKv* kv = taosArrayGetP(cols, i); - KvRowAppend(msg, kv->value, kv->valueLen, ¶m); + if(IS_VAR_DATA_TYPE(kv->type)){ + KvRowAppend(msg, kv->value, kv->length, ¶m); + }else{ + KvRowAppend(msg, &(kv->value), kv->length, ¶m); + } } *row = tdGetKVRowFromBuilder(tagsBuilder); @@ -1766,14 +1774,16 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols if (!kv || kv->length == 0) { MemRowAppend(&pBuf, NULL, 0, ¶m); } else { - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = kv->length; - } else if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { + int32_t colLen = kv->length; + if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); } - MemRowAppend(&pBuf, &(kv->value), colLen, ¶m); + if(IS_VAR_DATA_TYPE(kv->type)){ + MemRowAppend(&pBuf, kv->value, colLen, ¶m); + }else{ + MemRowAppend(&pBuf, &(kv->value), colLen, ¶m); + } } if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bc4fa9169c4bd7d93dd84a46544d9e2074eb84d5..6f9faaa6f00dccbbae42ee0df7167c45e553ae42 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1590,7 +1590,9 @@ static EDealRes checkStateExpr(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { STranslateContext* pCxt = pContext; SColumnNode* pCol = (SColumnNode*)pNode; - if (!IS_INTEGER_TYPE(pCol->node.resType.type)) { + + int32_t type = pCol->node.resType.type; + if (!IS_INTEGER_TYPE(type) && type != TSDB_DATA_TYPE_BOOL && !IS_VAR_DATA_TYPE(type)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); } if (COLUMN_TYPE_TAG == pCol->colType) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 676fe5dbfd5a11ea17217f0dc084e23436076216..0c703c9dfc40abd39a59673883c44806463a5e2b 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -91,7 +91,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_AGG_FUNC_NESTING: return "Aggregate functions do not support nesting"; case TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE: - return "Only support STATE_WINDOW on integer column"; + return "Only support STATE_WINDOW on integer/bool/varchar column"; case TSDB_CODE_PAR_INVALID_STATE_WIN_COL: return "Not support STATE_WINDOW on tag column"; case TSDB_CODE_PAR_INVALID_STATE_WIN_TABLE: diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 2652078b96452839a235384e7eace82e72a996f2..5962867869025190f6d8835f4dd558194d795fab 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -184,5 +184,6 @@ int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery) { if (TSDB_CODE_SUCCESS == code) { code = calculateConstant(pCxt, pQuery); } + return code; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 4b4c0796491b1c6565d9e359250c16f48257c15d..4c05358a13cbbf4195363b231f7854dcc0405ab3 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -97,7 +97,7 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTag static void* pTaskQueue = NULL; int32_t initTaskQueue() { - int32_t queueSize = tsMaxConnections * 2; + int32_t queueSize = tsMaxShellConns * 2; pTaskQueue = taosInitScheduler(queueSize, tsNumOfTaskQueueThreads, "tsc"); if (NULL == pTaskQueue) { qError("failed to init task queue"); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 822c214fe58bba136f773523da8ec7b0bb4ab768..fb9319bedeabfbd3673c72dda58ca0c3686cd940 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -290,6 +290,10 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STabl pTableMeta->sversion = msg->sversion; pTableMeta->tversion = msg->tversion; + if (isSuperTable) { + qDebug("stable %s meta returned, suid:%" PRIx64, msg->stbName, pTableMeta->suid); + } + pTableMeta->tableInfo.numOfTags = msg->numOfTags; pTableMeta->tableInfo.precision = msg->precision; pTableMeta->tableInfo.numOfColumns = msg->numOfColumns; diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 9257d2c0d49d2921044252ad9d5297cecb4fcdf0..99e61ad1db7b45725177255087ebc19a667c23e0 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -27,15 +27,18 @@ typedef struct SScalarCtx { SArray *pBlockList; /* element is SSDataBlock* */ SHashObj *pRes; /* element is SScalarParam */ void *param; // additional parameter (meta actually) for acquire value such as tbname/tags values + SHashObj *udf2Handle; } SScalarCtx; #define SCL_DATA_TYPE_DUMMY_HASH 9000 #define SCL_DEFAULT_OP_NUM 10 +#define SCL_DEFAULT_UDF_NUM 8 #define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type)) #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) -#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type) && (((SValueNode *)_node)->placeholderNo <= 0)) +//#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type) && (((SValueNode *)_node)->placeholderNo <= 0)) +#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type)) #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9843f0ac916e98ecc633d2ee8585cb41ebf458a0..8f4a9b96985afe52e6733d7a44683e30da878aa1 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -27,6 +27,7 @@ void sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { valueNode->datum.i = 0; } taosMemoryFree(timeStr); + valueNode->typeData = valueNode->datum.i; valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; @@ -153,6 +154,18 @@ void sclFreeRes(SHashObj *res) { taosHashCleanup(res); } +void sclFreeUdfHandles(SHashObj *udf2handle) { + void *pIter = taosHashIterate(udf2handle, NULL); + while (pIter) { + UdfcFuncHandle *handle = (UdfcFuncHandle *)pIter; + if (handle) { + teardownUdf(*handle); + } + pIter = taosHashIterate(udf2handle, pIter); + } + taosHashCleanup(udf2handle); +} + void sclFreeParam(SScalarParam *param) { if (param->columnData != NULL) { colDataDestroy(param->columnData); @@ -362,18 +375,24 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp if (fmIsUserDefinedFunc(node->funcId)) { UdfcFuncHandle udfHandle = NULL; - - code = setupUdf(node->functionName, &udfHandle); - if (code != 0) { - sclError("fmExecFunction error. setupUdf. function name: %s, code:%d", node->functionName, code); - goto _return; + char* udfName = node->functionName; + if (ctx->udf2Handle) { + UdfcFuncHandle *pHandle = taosHashGet(ctx->udf2Handle, udfName, strlen(udfName)); + if (pHandle) { + udfHandle = *pHandle; + } } - code = callUdfScalarFunc(udfHandle, params, paramNum, output); - if (code != 0) { - sclError("fmExecFunction error. callUdfScalarFunc. function name: %s, udf code:%d", node->functionName, code); - goto _return; + if (udfHandle == NULL) { + code = setupUdf(udfName, &udfHandle); + if (code != 0) { + sclError("fmExecFunction error. setupUdf. function name: %s, code:%d", udfName, code); + goto _return; + } + if (ctx->udf2Handle) { + taosHashPut(ctx->udf2Handle, udfName, strlen(udfName), &udfHandle, sizeof(UdfcFuncHandle)); + } } - code = teardownUdf(udfHandle); + code = callUdfScalarFunc(udfHandle, params, paramNum, output); if (code != 0) { sclError("fmExecFunction error. callUdfScalarFunc. function name: %s, udf code:%d", node->functionName, code); goto _return; @@ -891,15 +910,20 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { SScalarCtx ctx = {0}; ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { - sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); + sclError("taosHashInit result map failed, num:%d", SCL_DEFAULT_OP_NUM); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + ctx.udf2Handle = taosHashInit(SCL_DEFAULT_UDF_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == ctx.udf2Handle) { + sclError("taosHashInit udf to handle map failed, num:%d", SCL_DEFAULT_OP_NUM); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); SCL_ERR_JRET(ctx.code); *pRes = pNode; _return: + sclFreeUdfHandles(ctx.udf2Handle); sclFreeRes(ctx.pRes); return code; } @@ -915,10 +939,14 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { // TODO: OPT performance ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { - sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); + sclError("taosHashInit result map failed, num:%d", SCL_DEFAULT_OP_NUM); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + ctx.udf2Handle = taosHashInit(SCL_DEFAULT_UDF_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == ctx.udf2Handle) { + sclError("taosHashInit udf to handle map failed, num:%d", SCL_DEFAULT_OP_NUM); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); SCL_ERR_JRET(ctx.code); @@ -936,6 +964,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { _return: //nodesDestroyNode(pNode); + sclFreeUdfHandles(ctx.udf2Handle); sclFreeRes(ctx.pRes); return code; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index deb158cbaef59a70364bf6d4c87c8c0acb978b09..a3d1717c3bcabb22f910a821755c046fd7dcd6b0 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -92,7 +92,7 @@ int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) { syncRpcMsgLog2((char *)"==syncIOEqMsg==", pMsg); SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); STaosQueue *pMsgQ = queue; @@ -360,7 +360,7 @@ static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { syncRpcMsgLog2((char *)"==syncIOProcessRequest==", pMsg); SSyncIO *io = pParent; SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); taosWriteQitem(io->pMsgQ, pTemp); } @@ -420,7 +420,7 @@ static void syncIOTickQ(void *param, void *tmrId) { SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); syncRpcMsgLog2((char *)"==syncIOTickQ==", &rpcMsg); taosWriteQitem(io->pMsgQ, pTemp); diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 01490030f2f80383af39b521461ea3c47e732dd1..722f6bddef03cb6dbedb9d6e703db29c90e5a77c 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ # tdb -add_library(tdb STATIC "") +add_library(tdb SHARED "") target_sources(tdb PRIVATE "src/db/tdbPCache.c" diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index cd7e3beb1308d656f328a62e4d70d680f47c7c52..26f3f689aa4742ba115b697c55b7da9be1fa5b8b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -145,9 +145,9 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); } while (0) #define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) -#define CONN_PERSIST_TIME(para) (para * 1000 * 10) -#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_PERSIST_TIME(para) (para * 1000 * 10) +#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ @@ -223,11 +223,11 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); #define CONN_RELEASE_BY_SERVER(conn) \ (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) -#define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) -#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) +#define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) +#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) -#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) +#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) static void* cliWorkThread(void* arg); diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 3099998f57e74ff73b6af4fd8bd8e3882f0879ee..d8d78c2842cf9720575059986ce1ab2872cabfe1 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -114,7 +114,7 @@ int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 14d109dc5a2720ff723fdc485cb103514f90cf3a..80785340d13a21dcb01450a5a71d2d5509fe7fec 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -103,7 +103,7 @@ int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 0fe946bf68e669d4b5b152ec61e5b38c59883a34..e4d880f40a86da2c179151aaccc03a5849616edc 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -36,7 +36,7 @@ int64_t interlocked_add_fetch_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_add_fetch_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)(_InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)val) + (int32_t)val); #else return (void*)(InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)val) + (int64_t)val); @@ -56,7 +56,7 @@ int32_t interlocked_and_fetch_32(int32_t volatile* ptr, int32_t val) { } int64_t interlocked_and_fetch_64(int64_t volatile* ptr, int64_t val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS int64_t old, res; do { old = *ptr; @@ -69,7 +69,7 @@ int64_t interlocked_and_fetch_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_and_fetch_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)interlocked_and_fetch_32((int32_t volatile*)ptr, (int32_t)val); #else return (void*)interlocked_and_fetch_64((int64_t volatile*)ptr, (int64_t)val); @@ -77,7 +77,7 @@ void* interlocked_and_fetch_ptr(void* volatile* ptr, void* val) { } int64_t interlocked_fetch_and_64(int64_t volatile* ptr, int64_t val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS int64_t old; do { old = *ptr; @@ -89,7 +89,7 @@ int64_t interlocked_fetch_and_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_fetch_and_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)_InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); #else return (void*)_InterlockedAnd64((int64_t volatile*)(ptr), (int64_t)(val)); @@ -109,7 +109,7 @@ int32_t interlocked_or_fetch_32(int32_t volatile* ptr, int32_t val) { } int64_t interlocked_or_fetch_64(int64_t volatile* ptr, int64_t val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS int64_t old, res; do { old = *ptr; @@ -122,7 +122,7 @@ int64_t interlocked_or_fetch_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_or_fetch_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)interlocked_or_fetch_32((int32_t volatile*)ptr, (int32_t)val); #else return (void*)interlocked_or_fetch_64((int64_t volatile*)ptr, (int64_t)val); @@ -130,7 +130,7 @@ void* interlocked_or_fetch_ptr(void* volatile* ptr, void* val) { } int64_t interlocked_fetch_or_64(int64_t volatile* ptr, int64_t val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS int64_t old; do { old = *ptr; @@ -142,7 +142,7 @@ int64_t interlocked_fetch_or_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_fetch_or_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)_InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); #else return (void*)interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); @@ -162,7 +162,7 @@ int32_t interlocked_xor_fetch_32(int32_t volatile* ptr, int32_t val) { } int64_t interlocked_xor_fetch_64(int64_t volatile* ptr, int64_t val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS int64_t old, res; do { old = *ptr; @@ -175,7 +175,7 @@ int64_t interlocked_xor_fetch_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_xor_fetch_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #else return (void*)interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); @@ -183,7 +183,7 @@ void* interlocked_xor_fetch_ptr(void* volatile* ptr, void* val) { } int64_t interlocked_fetch_xor_64(int64_t volatile* ptr, int64_t val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS int64_t old; do { old = *ptr; @@ -195,7 +195,7 @@ int64_t interlocked_fetch_xor_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_fetch_xor_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)_InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); #else return (void*)interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); @@ -211,7 +211,7 @@ int64_t interlocked_sub_fetch_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_sub_fetch_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)interlocked_sub_fetch_32((int32_t volatile*)ptr, (int32_t)val); #else return (void*)interlocked_add_fetch_64((int64_t volatile*)ptr, (int64_t)val); @@ -226,7 +226,7 @@ int64_t interlocked_fetch_sub_64(int64_t volatile* ptr, int64_t val) { } void* interlocked_fetch_sub_ptr(void* volatile* ptr, void* val) { -#ifdef _TD_WINDOWS_32 +#ifdef WINDOWS return (void*)interlocked_fetch_sub_32((int32_t volatile*)ptr, (int32_t)val); #else return (void*)interlocked_fetch_sub_64((int64_t volatile*)ptr, (int64_t)val); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 425bf8b7ac220ac92a7c490377fde283e541de9a..3cd05b65cd2e00f2f7308ac8e8fb1de7d1b7f9a8 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -109,6 +109,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha int64_t taosCopyFile(const char *from, const char *to) { #ifdef WINDOWS + assert(0); return 0; #else char buffer[4096]; @@ -152,16 +153,16 @@ int32_t taosRemoveFile(const char *path) { return remove(path); } int32_t taosRenameFile(const char *oldName, const char *newName) { #ifdef WINDOWS - int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); - if (code < 0) { - // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + bool code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + if (!code) { + printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } - return code; + return !code; #else int32_t code = rename(oldName, newName); if (code < 0) { - // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } return code; @@ -169,11 +170,12 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { } int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { + struct stat fileStat; #ifdef WINDOWS - return 0; + int32_t code = _stat(path, &fileStat); #else - struct stat fileStat; int32_t code = stat(path, &fileStat); +#endif if (code < 0) { return code; } @@ -187,14 +189,15 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { } return 0; -#endif } int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { + + struct stat fileStat; #ifdef WINDOWS - return 0; + int32_t code = _stat(path, &fileStat); #else - struct stat fileStat; int32_t code = stat(path, &fileStat); +#endif if (code < 0) { return code; } @@ -208,7 +211,6 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { } return 0; -#endif } void autoDelFileListAdd(const char *path) { return; } @@ -276,9 +278,6 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } int64_t taosCloseFile(TdFilePtr *ppFile) { -#ifdef WINDOWS - return 0; -#else if (ppFile == NULL || *ppFile == NULL) { return 0; } @@ -294,7 +293,12 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { (*ppFile)->fp = NULL; } if ((*ppFile)->fd >= 0) { + #ifdef WINDOWS + HANDLE h = (HANDLE)_get_osfhandle((*ppFile)->fd); + !FlushFileBuffers(h); + #else fsync((*ppFile)->fd); + #endif close((*ppFile)->fd); (*ppFile)->fd = -1; } @@ -306,7 +310,6 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { taosMemoryFree(*ppFile); *ppFile = NULL; return 0; -#endif } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { @@ -412,13 +415,17 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { -#ifdef WINDOWS - return 0; -#else + if (pFile == NULL) { + return 0; + } assert(pFile->fd >= 0); // Please check if you have closed the file. struct stat fileStat; +#ifdef WINDOWS + int32_t code = _fstat(pFile->fd, &fileStat); +#else int32_t code = fstat(pFile->fd, &fileStat); +#endif if (code < 0) { return code; } @@ -432,7 +439,6 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { } return 0; -#endif } int32_t taosLockFile(TdFilePtr pFile) { @@ -459,7 +465,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { #ifdef WINDOWS if (pFile->fd < 0) { errno = EBADF; - printf("%s\n", "fd arg was negative"); + printf("Ftruncate file error, fd arg was negative\n"); return -1; } @@ -516,26 +522,20 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { } int32_t taosFsyncFile(TdFilePtr pFile) { -#ifdef WINDOWS - if (pFile->fd < 0) { - errno = EBADF; - printf("%s\n", "fd arg was negative"); - return -1; - } - - HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); - - return !FlushFileBuffers(h); -#else if (pFile == NULL) { return 0; } if (pFile->fp != NULL) return fflush(pFile->fp); - if (pFile->fd >= 0) return fsync(pFile->fd); - + if (pFile->fd >= 0) { + #ifdef WINDOWS + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + return !FlushFileBuffers(h); + #else + return fsync(pFile->fd); + #endif + } return 0; -#endif } int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 7c877b463a785f46e4b04694462539bcadeeb405..e3791af618d341d1cd08a5fc46c9d62055be2e13 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -138,7 +138,7 @@ static void print_line(Dwarf_Debug dbg, Dwarf_Line line, Dwarf_Addr pc) { dwarf_linesrc(line, &linesrc, NULL); dwarf_lineno(line, &lineno, NULL); } - printf("%s:%" DW_PR_DUu "\n", linesrc, lineno); + printf("BackTrace %08" PRId64 " %s:%" DW_PR_DUu "\n", taosGetSelfPthreadId(), linesrc, lineno); if (line) dwarf_dealloc(dbg, linesrc, DW_DLA_STRING); } void taosPrintBackTrace() { diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index f92a3b37836a787ac17f870ab0377ce54d369523..74f1356abf3e24a7b997571989471da761273c4d 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -19,6 +19,7 @@ int32_t taosNewProc(char **args) { #ifdef WINDOWS + assert(0); return 0; #else int32_t pid = fork(); @@ -36,6 +37,7 @@ int32_t taosNewProc(char **args) { void taosWaitProc(int32_t pid) { #ifdef WINDOWS + assert(0); #else int32_t status = -1; waitpid(pid, &status, 0); @@ -44,6 +46,7 @@ void taosWaitProc(int32_t pid) { void taosKillProc(int32_t pid) { #ifdef WINDOWS + assert(0); #else kill(pid, SIGINT); #endif @@ -51,6 +54,7 @@ void taosKillProc(int32_t pid) { bool taosProcExist(int32_t pid) { #ifdef WINDOWS + assert(0); return false; #else int32_t p = getpgid(pid); diff --git a/source/os/src/osShm.c b/source/os/src/osShm.c index 1cd51f94a09f770914c67e46cdb65f6334d83b6c..cb09e2fb38c5edcc00ae753245d98b9ca498e825 100644 --- a/source/os/src/osShm.c +++ b/source/os/src/osShm.c @@ -23,6 +23,7 @@ static int32_t shmids[MAX_SHMIDS] = {0}; static void taosDeleteCreatedShms() { #if defined(WINDOWS) + assert(0); #else for (int32_t i = 0; i < MAX_SHMIDS; ++i) { int32_t shmid = shmids[i] - 1; @@ -35,6 +36,7 @@ static void taosDeleteCreatedShms() { int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { #if defined(WINDOWS) + assert(0); #else pShm->id = -1; @@ -75,6 +77,7 @@ int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { void taosDropShm(SShm* pShm) { #if defined(WINDOWS) + assert(0); #else if (pShm->id >= 0) { if (pShm->ptr != NULL) { @@ -90,6 +93,7 @@ void taosDropShm(SShm* pShm) { int32_t taosAttachShm(SShm* pShm) { #if defined(WINDOWS) + assert(0); #else errno = 0; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 2410586287952e690349f44d84f928d98962c804..105acb188a4236b6889cfb1c26d08479ea32f387 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -285,6 +285,7 @@ int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return -1; } #ifdef WINDOWS + assert(0); return 0; #else return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); @@ -642,6 +643,7 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { int taosGetLocalIp(const char *eth, char *ip) { #if defined(WINDOWS) // DO NOTHAING + assert(0); return 0; #else int fd; @@ -668,6 +670,7 @@ int taosGetLocalIp(const char *eth, char *ip) { int taosValidIp(uint32_t ip) { #if defined(WINDOWS) // DO NOTHAING + assert(0); return 0; #else int ret = -1; @@ -866,6 +869,7 @@ int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len void taosBlockSIGPIPE() { #ifdef WINDOWS + // assert(0); #else sigset_t signal_mask; sigemptyset(&signal_mask); @@ -976,14 +980,12 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { } void taosIgnSIGPIPE() { -#ifdef WINDOWS -#else signal(SIGPIPE, SIG_IGN); -#endif } void taosSetMaskSIGPIPE() { #ifdef WINDOWS + // assert(0); #else sigset_t signal_mask; sigemptyset(&signal_mask); @@ -1005,6 +1007,7 @@ int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *a TdEpollPtr taosCreateEpoll(int32_t size) { EpollFd fd = -1; #ifdef WINDOWS + assert(0); #else fd = epoll_create(size); #endif @@ -1027,6 +1030,7 @@ int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocke return -1; } #ifdef WINDOWS + assert(0); #else code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event); #endif @@ -1038,6 +1042,7 @@ int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxE return -1; } #ifdef WINDOWS + assert(0); #else code = epoll_wait(pEpoll->fd, event, maxEvents, timeout); #endif diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 375c5001f4a1025cdaa44e317e57548170022fd5..7dbd3019131073d60db594fdbbfda954fcc393d6 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -236,3 +236,22 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { ret int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); } int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); } + +char *taosStrCaseStr(const char *str, const char *pattern) { + size_t i; + + if (!*pattern) + return (char*)str; + + for (; *str; str++) { + if (toupper(*str) == toupper(*pattern)) { + for (i = 1;; i++) { + if (!pattern[i]) + return (char*)str; + if (toupper(str[i]) != toupper(pattern[i])) + break; + } + } + } + return NULL; +} \ No newline at end of file diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 348424b37246222cfdb74dcff0513c0f2a5711e9..fd6172e04fe8d034d4f0b4a0eb5b56f6416db10d 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -129,7 +129,21 @@ static void taosGetProcIOnfos() { static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { #ifdef WINDOWS + FILETIME pre_idleTime = {0}; + FILETIME pre_kernelTime = {0}; + FILETIME pre_userTime = {0}; + FILETIME idleTime; + FILETIME kernelTime; + FILETIME userTime; + bool res = GetSystemTimes(&idleTime, &kernelTime, &userTime); + if (res) { + cpuInfo->idle = CompareFileTime(&pre_idleTime, &idleTime); + cpuInfo->system = CompareFileTime(&pre_kernelTime, &kernelTime); + cpuInfo->user = CompareFileTime(&pre_userTime, &userTime); + cpuInfo->nice = 0; + } #elif defined(_TD_DARWIN_64) + assert(0); #else TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { @@ -155,7 +169,18 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { #ifdef WINDOWS + FILETIME pre_krnlTm = {0}; + FILETIME pre_usrTm = {0}; + FILETIME creatTm, exitTm, krnlTm, usrTm; + + if (GetThreadTimes(GetCurrentThread(), &creatTm, &exitTm, &krnlTm, &usrTm)) { + cpuInfo->stime = CompareFileTime(&pre_krnlTm, &krnlTm); + cpuInfo->utime = CompareFileTime(&pre_usrTm, &usrTm); + cpuInfo->cutime = 0; + cpuInfo->cstime = 0; + } #elif defined(_TD_DARWIN_64) + assert(0); #else TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { @@ -219,6 +244,7 @@ void taosGetSystemInfo() { int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS + // assert(0); #elif defined(_TD_DARWIN_64) const char *filepath = "/usr/local/taos/email"; @@ -250,6 +276,7 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { #ifdef WINDOWS + assert(0); #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -305,6 +332,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { #ifdef WINDOWS + assert(0); #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -716,9 +744,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { #ifdef WINDOWS GUID guid; CoCreateGuid(&guid); - - sprintf(uid, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0], - guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); + memcpy(uid, &guid, uidlen); return 0; #elif defined(_TD_DARWIN_64) @@ -750,6 +776,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { char *taosGetCmdlineByPID(int pid) { #ifdef WINDOWS + assert(0); return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 62c1747619154257c68bac98bf1e26b1b38d25c5..ba07b6c3dd8e8d56e886e176bc14063a2db464e9 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -33,6 +33,7 @@ typedef struct FILE TdCmd; void* taosLoadDll(const char* filename) { #if defined(WINDOWS) + assert(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -51,6 +52,7 @@ void* taosLoadDll(const char* filename) { void* taosLoadSym(void* handle, char* name) { #if defined(WINDOWS) + assert(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -71,6 +73,7 @@ void* taosLoadSym(void* handle, char* name) { void taosCloseDll(void* handle) { #if defined(WINDOWS) + assert(0); return; #elif defined(_TD_DARWIN_64) return; @@ -121,6 +124,7 @@ int taosSetConsoleEcho(bool on) { void taosSetTerminalMode() { #if defined(WINDOWS) + // assert(0); #else struct termios newtio; @@ -154,7 +158,7 @@ void taosSetTerminalMode() { int32_t taosGetOldTerminalMode() { #if defined(WINDOWS) - + // assert(0); #else /* Make sure stdin is a terminal. */ if (!isatty(STDIN_FILENO)) { @@ -172,7 +176,7 @@ int32_t taosGetOldTerminalMode() { void taosResetTerminalMode() { #if defined(WINDOWS) - + // assert(0); #else if (tcsetattr(0, TCSANOW, &oldtio) != 0) { fprintf(stderr, "Fail to reset the terminal properties!\n"); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8f48e0585e79294dd65649a37a12a2fbcb17beb4..11d7d9831aadf9f419e618d745c70066147bfa45 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -640,13 +640,14 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { } int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { - char *buf, *name, *value, *value2, *value3; + char buf[1024], *name, *value, *value2, *value3; int32_t olen, vlen, vlen2, vlen3; int32_t index = 0; if (envCmd == NULL) return 0; while (envCmd[index]!=NULL) { - buf = taosMemoryMalloc(strlen(envCmd[index])); - taosEnvToCfg(envCmd[index], buf); + strncpy(buf, envCmd[index], sizeof(buf)-1); + buf[sizeof(buf)-1] = 0; + taosEnvToCfg(buf, buf); index++; name = value = value2 = value3 = NULL; @@ -671,8 +672,6 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD); } - - taosMemoryFree(buf); } uInfo("load from env cmd cfg success"); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a1bc37e6cd1231a6f5486d526756beb5916f45e8..6c0a9b132480944c082c4c7e0e8e4abd9ac4727d 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -95,6 +95,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") TAOS_DEFINE_ERROR(TSDB_CODE_DUP_KEY, "Cannot add duplicate keys to hash") TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed") +TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE, "Out of memory in rpc queue") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 8963a4f94eb23799374a355f9a44a24a6962e2e5..8b4fd235fddb1ea963053485a099b206c51d8e69 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -258,8 +258,8 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea int16_t headLen = CEIL8(rawHeadLen); int32_t bodyLen = CEIL8(rawBodyLen); - void *pHead = (*mallocHeadFp)(headLen); - void *pBody = (*mallocBodyFp)(bodyLen); + void *pHead = (*mallocHeadFp)(headLen, RPC_QITEM); + void *pBody = (*mallocBodyFp)(bodyLen, RPC_QITEM); if (pHead == NULL || pBody == NULL) { taosThreadMutexUnlock(&pQueue->mutex); tsem_post(&pQueue->sem); diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 3c3a8460b96041538c3a625812982c391c91bdf5..17a87b1a5fffa77c076c2201dcdd18312a09c488 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -18,41 +18,45 @@ #include "taoserror.h" #include "tlog.h" +int64_t tsRpcQueueMemoryAllowed = 0; +int64_t tsRpcQueueMemoryUsed = 0; + typedef struct STaosQnode STaosQnode; typedef struct STaosQnode { STaosQnode *next; STaosQueue *queue; + int32_t size; + int8_t itype; + int8_t reserved[3]; char item[]; } STaosQnode; typedef struct STaosQueue { - int32_t itemSize; - int32_t numOfItems; - int32_t threadId; - STaosQnode *head; - STaosQnode *tail; - STaosQueue *next; // for queue set - STaosQset *qset; // for queue set - void *ahandle; // for queue set - FItem itemFp; - FItems itemsFp; + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set + FItem itemFp; + FItems itemsFp; TdThreadMutex mutex; + int64_t memOfItems; + int32_t numOfItems; } STaosQueue; typedef struct STaosQset { - STaosQueue *head; - STaosQueue *current; + STaosQueue *head; + STaosQueue *current; TdThreadMutex mutex; - int32_t numOfQueues; - int32_t numOfItems; - tsem_t sem; + tsem_t sem; + int32_t numOfQueues; + int32_t numOfItems; } STaosQset; typedef struct STaosQall { STaosQnode *current; STaosQnode *start; - int32_t itemSize; int32_t numOfItems; } STaosQall; @@ -118,32 +122,61 @@ bool taosQueueEmpty(STaosQueue *queue) { return empty; } -int32_t taosQueueSize(STaosQueue *queue) { +int32_t taosQueueItemSize(STaosQueue *queue) { + if (queue == NULL) return 0; + taosThreadMutexLock(&queue->mutex); int32_t numOfItems = queue->numOfItems; taosThreadMutexUnlock(&queue->mutex); return numOfItems; } -void *taosAllocateQitem(int32_t size) { +int64_t taosQueueMemorySize(STaosQueue *queue) { + if (queue == NULL) return 0; + + taosThreadMutexLock(&queue->mutex); + int64_t memOfItems = queue->memOfItems; + taosThreadMutexUnlock(&queue->mutex); + return memOfItems; +} + +void *taosAllocateQitem(int32_t size, EQItype itype) { STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); + pNode->size = size; + pNode->itype = itype; if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - uTrace("item:%p, node:%p is allocated", pNode->item, pNode); + if (itype == RPC_QITEM) { + int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size); + if (alloced > tsRpcQueueMemoryUsed) { + taosMemoryFree(pNode); + terrno = TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE; + return NULL; + } + uTrace("item:%p, node:%p is allocated, alloc:%" PRId64, pNode->item, pNode, alloced); + } else { + uTrace("item:%p, node:%p is allocated", pNode->item, pNode); + } + return (void *)pNode->item; } void taosFreeQitem(void *pItem) { if (pItem == NULL) return; - char *temp = pItem; - temp -= sizeof(STaosQnode); - uTrace("item:%p, node:%p is freed", pItem, temp); - taosMemoryFree(temp); + STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); + if (pNode->itype > 0) { + int64_t alloced = atomic_sub_fetch_64(&tsRpcQueueMemoryUsed, pNode->size); + uTrace("item:%p, node:%p is freed, alloc:%" PRId64, pItem, pNode, alloced); + } else { + uTrace("item:%p, node:%p is freed", pItem, pNode); + } + + taosMemoryFree(pNode); } void taosWriteQitem(STaosQueue *queue, void *pItem) { @@ -161,8 +194,9 @@ void taosWriteQitem(STaosQueue *queue, void *pItem) { } queue->numOfItems++; + queue->memOfItems += pNode->size; if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); - uTrace("item:%p is put into queue:%p, items:%d", pItem, queue, queue->numOfItems); + uTrace("item:%p is put into queue:%p, items:%d mem:%" PRId64, pItem, queue, queue->numOfItems, queue->memOfItems); taosThreadMutexUnlock(&queue->mutex); @@ -181,9 +215,11 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; + queue->memOfItems -= pNode->size; if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, 1); code = 1; - uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, items:%d mem:%" PRId64, *ppItem, queue, queue->numOfItems, + queue->memOfItems); } taosThreadMutexUnlock(&queue->mutex); @@ -191,7 +227,13 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { return code; } -STaosQall *taosAllocateQall() { return taosMemoryCalloc(1, sizeof(STaosQall)); } +STaosQall *taosAllocateQall() { + STaosQall *qall = taosMemoryCalloc(1, sizeof(STaosQall)); + if (qall != NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return qall; +} void taosFreeQall(STaosQall *qall) { taosMemoryFree(qall); } @@ -207,12 +249,12 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { qall->current = queue->head; qall->start = queue->head; qall->numOfItems = queue->numOfItems; - qall->itemSize = queue->itemSize; code = qall->numOfItems; queue->head = NULL; queue->tail = NULL; queue->numOfItems = 0; + queue->memOfItems = 0; if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, qall->numOfItems); } @@ -377,9 +419,11 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FI queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; + queue->memOfItems -= pNode->size; atomic_sub_fetch_32(&qset->numOfItems, 1); code = 1; - uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, items:%d mem:%" PRId64, *ppItem, queue, queue->numOfItems, + queue->memOfItems); } taosThreadMutexUnlock(&queue->mutex); @@ -411,7 +455,6 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand qall->current = queue->head; qall->start = queue->head; qall->numOfItems = queue->numOfItems; - qall->itemSize = queue->itemSize; code = qall->numOfItems; if (ahandle) *ahandle = queue->ahandle; if (itemsFp) *itemsFp = queue->itemsFp; @@ -419,6 +462,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand queue->head = NULL; queue->tail = NULL; queue->numOfItems = 0; + queue->memOfItems = 0; atomic_sub_fetch_32(&qset->numOfItems, qall->numOfItems); for (int32_t j = 1; j < qall->numOfItems; ++j) { tsem_wait(&qset->sem); @@ -444,23 +488,3 @@ void taosResetQsetThread(STaosQset *qset, void *pItem) { } taosThreadMutexUnlock(&qset->mutex); } - -int32_t taosGetQueueItemsNumber(STaosQueue *queue) { - if (!queue) return 0; - - int32_t num; - taosThreadMutexLock(&queue->mutex); - num = queue->numOfItems; - taosThreadMutexUnlock(&queue->mutex); - return num; -} - -int32_t taosGetQsetItemsNumber(STaosQset *qset) { - if (!qset) return 0; - - int32_t num = 0; - taosThreadMutexLock(&qset->mutex); - num = qset->numOfItems; - taosThreadMutexUnlock(&qset->mutex); - return num; -} diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 3c23e784c5e7eca90ea60b01f35a0a396c8dcccc..c78efb4e8dbcc95c71f5a863fcd93413fcdf1736 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -30,17 +30,12 @@ class TDSimClient: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "minTablesPerVnode": "4", - "maxTablesPerVnode": "1000", - "tableIncStepPerVnode": "10000", - "maxVgroupsPerDb": "1000", - "sdbDebugFlag": "143", "rpcDebugFlag": "143", "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", + "cDebugFlag": "143", + "udebugFlag": "143", + "jnidebugFlag": "143", + "qdebugFlag": "143", "telemetryReporting": "0", } @@ -115,36 +110,29 @@ class TDDnode: self.testCluster = False self.valgrind = 0 self.cfgDict = { - "numOfLogLines": "100000000", - "mnodeEqualVnodeNum": "0", "walLevel": "2", "fsync": "1000", - "statusInterval": "1", - "numOfMnodes": "3", - "numOfThreadsPerCore": "2.0", "monitor": "0", - "maxVnodeConnections": "30000", - "maxMgmtConnections": "30000", - "maxMeterConnections": "30000", "maxShellConns": "30000", "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "anyIp": "0", - "telemetryReporting": "0", - "dDebugFlag": "135", - "tsdbDebugFlag": "135", - "mDebugFlag": "135", - "sdbDebugFlag": "135", + "mDebugFlag": "143", + "dDebugFlag": "143", + "vDebugFlag": "143", + "tqDebugFlag": "143", + "cDebugFlag": "143", + "jniDebugFlag": "143", + "qDebugFlag": "143", "rpcDebugFlag": "143", "tmrDebugFlag": "131", - "cDebugFlag": "135", - "httpDebugFlag": "135", - "monitorDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "maxSQLLength": "1048576" + "uDebugFlag": "143", + "sDebugFlag": "135", + "wDebugFlag": "143", + "qdebugFlag": "143", + "numOfLogLines": "100000000", + "statusInterval": "1", + "telemetryReporting": "0" } def init(self, path): diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index ae07b3fc8ecd54b048d1429b43e0a2406fcd96a9..cb914f8d6dbe6b18a0a5c45084e7bd6fdc1059d5 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -9,10 +9,12 @@ #include #include "../../../include/client/taos.h" +#define FUNCTION_TEST_IDX 1 + int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; -int32_t optrIdxList[] = {0, 9}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; +int32_t optrIdxList[] = {0, 7}; typedef struct { char* oper; @@ -53,7 +55,6 @@ FuncInfo funcInfo[] = { {"count", 1}, {"sum", 1}, {"min", 1}, - {"sin", 1}, }; char *bpStbPrefix = "st"; @@ -66,6 +67,10 @@ int32_t bpDefaultStbId = 1; //char *varoperatorList[] = {">", ">=", "<", "<=", "=", "<>", "in", "not in", "like", "not like", "match", "nmatch"}; #define tListLen(x) (sizeof(x) / sizeof((x)[0])) +#define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) +#define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) +#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) +#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) typedef struct { int64_t* tsData; @@ -165,8 +170,11 @@ CaseCfg gCase[] = { // 22 {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, - {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 2, 10, 1, 3, 0, 0, 1, 2}, +// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, +// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + + {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, + {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, }; @@ -181,6 +189,7 @@ typedef struct { bool printQuerySql; bool printStmtSql; bool autoCreateTbl; + bool numericParam; int32_t rowNum; //row num for one table int32_t bindColNum; int32_t bindTagNum; @@ -207,6 +216,7 @@ CaseCtrl gCaseCtrl = { // default .printQuerySql = true, .printStmtSql = true, .autoCreateTbl = false, + .numericParam = false, .rowNum = 0, .bindColNum = 0, .bindTagNum = 0, @@ -259,26 +269,22 @@ CaseCtrl gCaseCtrl = { #if 1 CaseCtrl gCaseCtrl = { // query case with specified col&oper - .bindNullNum = 0, + .bindNullNum = 1, .printCreateTblSql = false, .printQuerySql = true, .printStmtSql = true, .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, - .bindColTypeNum = 0, - .bindColTypeList = NULL, - .optrIdxListNum = 0, - .optrIdxList = NULL, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, .checkParamNum = false, .printRes = true, .runTimes = 0, .caseRunIdx = -1, - .optrIdxListNum = 0, - .optrIdxList = NULL, - .bindColTypeNum = 0, - .bindColTypeList = NULL, - .caseIdx = 24, + .caseIdx = 23, .caseNum = 1, .caseRunNum = 1, }; @@ -303,11 +309,11 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .printRes = true, .runTimes = 0, .caseRunIdx = -1, - //.optrIdxListNum = tListLen(optrIdxList), - //.optrIdxList = optrIdxList, - //.bindColTypeNum = tListLen(bindColTypeList), - //.bindColTypeList = bindColTypeList, - .caseIdx = 22, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseIdx = 24, .caseNum = 1, .caseRunNum = 1, }; @@ -661,11 +667,11 @@ void bpGenerateConstInFuncSQL(BindData *data, int32_t tblIdx) { void generateQueryMiscSQL(BindData *data, int32_t tblIdx) { switch(tblIdx) { case 0: - bpGenerateConstInOpSQL(data, tblIdx); - break; - case 1: //TODO FILL TEST default: + bpGenerateConstInOpSQL(data, tblIdx); + break; + case FUNCTION_TEST_IDX: bpGenerateConstInFuncSQL(data, tblIdx); break; } @@ -709,6 +715,16 @@ void generateColDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_ } else if (gCurCase->fullCol) { *dataType = gCurCase->colList[bindIdx]; return; + } else if (gCaseCtrl.numericParam) { + while (true) { + *dataType = rand() % (TSDB_DATA_TYPE_MAX - 1) + 1; + if (!IS_NUMERIC_TYPE(*dataType)) { + continue; + } + + break; + } + return; } else if (0 == colIdx) { *dataType = TSDB_DATA_TYPE_TIMESTAMP; return; @@ -1046,6 +1062,10 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { data->binaryLen[i] = gVarCharLen; } + if (tblIdx == FUNCTION_TEST_IDX) { + gCaseCtrl.numericParam = true; + } + for (int b = 0; b < bindNum; b++) { for (int c = 0; c < gCurCase->bindColNum; ++c) { prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 19e133e9496ea6161656d830432049eebe407e8c..b02ca79ed46451a6dcaf6c45ef81598715f749fa 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -94,6 +94,44 @@ endi if $data00 != 2.645751311 then return -1 endi + +sql insert into t2 values(now+4s, 4, 8)(now+5s, 5, 9); +sql select udf2(f1-f2), udf2(f1+f2) from t2; +print $rows , $data00 , $data01 +if $rows != 1 then + return -1; +endi +if $data00 != 5.656854249 then + return -1 +endi +if $data01 != 18.547236991 then + return -1 +endi + +sql select udf2(udf1(f2-f1)), udf2(udf1(f2/f1)) from t2; +print $rows , $data00 , $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 176.000000000 then + return -1 +endi +if $data01 != 152.420471066 then + return -1 +endi + +sql select udf2(f2) from udf.t2 group by 1-udf1(f1); +print $rows , $data00 , $data10 +if $rows != 2 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi +if $data10 != 12.083045974 then + return -1 +endi + sql drop function udf1; sql show functions; if $rows != 1 then diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 289dd3d62df0b38fc3d6d857ba9bcd22bfd14aae..8fc131e58173faf31fcc4ffbc8fab08f6e937aea 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py new file mode 100644 index 0000000000000000000000000000000000000000..b8d3abca5c1cb8828777ff434c82ec8f3b659e30 --- /dev/null +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -0,0 +1,400 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg,showRow,cdbName,valgrind=0): + shellCmd = 'nohup ' + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("use %s" %dbName) + tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + event.set() + tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + print ("input parameters:") + print (parameterDict) + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + self.create_tables(tsql,\ + parameterDict["dbName"],\ + parameterDict["vgroups"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"]) + + self.insert_data(tsql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"],\ + parameterDict["startTs"]) + return + + def tmqCase1(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db1', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 100000, \ + 'batchNum': 200, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + + tdLog.info("create consume info table and consume result table") + cdbName = parameterDict["dbName"] + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) + tdSql.query(sql) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 1: + break + else: + time.sleep(5) + + tdLog.info("consumer result: %d, %d"%(tdSql.getData(0 , 2), tdSql.getData(0 , 3))) + tdSql.checkData(0 , 1, consumerId) + # mulit rows and mulit tables in one sql, this num of msg is not sure + #tdSql.checkData(0 , 2, expectmsgcnt) + tdSql.checkData(0 , 3, expectrowcnt+1) + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db2', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 100000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + + tdLog.info("create consume info table and consume result table") + cdbName = parameterDict["dbName"] + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) + tdSql.query(sql) + + consumerId = 1 + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) + tdSql.query(sql) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 2: + break + else: + time.sleep(5) + + consumerId0 = tdSql.getData(0 , 1) + consumerId1 = tdSql.getData(1 , 1) + actConsumeRows0 = tdSql.getData(0 , 3) + actConsumeRows1 = tdSql.getData(1 , 3) + + tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0)) + tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1)) + + totalConsumeRows = actConsumeRows0 + actConsumeRows1 + if totalConsumeRows != expectrowcnt + 2: + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def tmqCase3(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 3: Produce while one consumers to subscribe one db, include 2 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db3', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 100000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + parameterDict2 = {'cfg': '', \ + 'dbName': 'db3', \ + 'vgroups': 4, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 100000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + + tdLog.info("create consume info table and consume result table") + cdbName = parameterDict["dbName"] + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) + tdSql.query(sql) + + # consumerId = 1 + # sql = "insert into %s.consumeinfo values "%cdbName + # sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) + # tdSql.query(sql) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + + # wait for data ready + prepareEnvThread.join() + prepareEnvThread2.join() + + tdLog.info("insert process end, and start to check consume result") + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 1: + break + else: + time.sleep(5) + + consumerId0 = tdSql.getData(0 , 1) + #consumerId1 = tdSql.getData(1 , 1) + actConsumeRows0 = tdSql.getData(0 , 3) + #actConsumeRows1 = tdSql.getData(1 , 3) + + tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0)) + #tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1)) + + #totalConsumeRows = actConsumeRows0 + actConsumeRows1 + if actConsumeRows0 != expectrowcnt + 1: + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 3 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + #self.tmqCase1(cfgPath, buildPath) + self.tmqCase2(cfgPath, buildPath) + #self.tmqCase3(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/99-TDcase/TD-15554.py b/tests/system-test/99-TDcase/TD-15554.py index 890580ca2ce6a7e6a8e5e0fa0f2950f0002bf340..d7b2856b417ccf57394e504fd3ef17c42e7138d0 100644 --- a/tests/system-test/99-TDcase/TD-15554.py +++ b/tests/system-test/99-TDcase/TD-15554.py @@ -14,10 +14,10 @@ from util.dnodes import * class TDTestCase: hostname = socket.gethostname() - clientCfgDict = {'qdebugflag':'143'} - updatecfgDict = {'clientCfg': {}, 'qdebugflag':'143'} - updatecfgDict["clientCfg"] = clientCfgDict - print ("===================: ", updatecfgDict) + #clientCfgDict = {'qdebugflag':'143'} + #updatecfgDict = {'clientCfg': {}, 'qdebugflag':'143'} + #updatecfgDict["clientCfg"] = clientCfgDict + #print ("===================: ", updatecfgDict) def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") diff --git a/tests/system-test/99-TDcase/TD-15563.py b/tests/system-test/99-TDcase/TD-15563.py new file mode 100644 index 0000000000000000000000000000000000000000..5931360b905aa4c7f75bb1bc68402224001c1270 --- /dev/null +++ b/tests/system-test/99-TDcase/TD-15563.py @@ -0,0 +1,392 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + shellCmd = 'nohup ' + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("use %s" %dbName) + tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + event.set() + tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + print ("input parameters:") + print (parameterDict) + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + self.create_tables(tsql,\ + parameterDict["dbName"],\ + parameterDict["vgroups"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"]) + + self.insert_data(tsql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"],\ + parameterDict["startTs"]) + return + + def tmqCase1(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db, inclue 1 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db1', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db, inclue 1 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db2', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + consumerId = 1 + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def tmqCase3(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 3: Produce while one consumers to subscribe one db, include 2 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db3', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + parameterDict2 = {'cfg': '', \ + 'dbName': 'db3', \ + 'vgroups': 4, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + # consumerId = 1 + # self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + # wait for data ready + prepareEnvThread.join() + prepareEnvThread2.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 3 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase1(cfgPath, buildPath) + #self.tmqCase2(cfgPath, buildPath) + self.tmqCase3(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 33ddd23d8c407995fffc9ba07278dc6f2de28332..1228d6174c0e1ff672cc7d6fc6cb64729b50b280 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -179,7 +179,7 @@ void parseArgument(int32_t argc, char* argv[]) { } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else { - printf("%s unknow para: %s %s", GREEN, argv[++i], NC); + pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); } } @@ -259,7 +259,7 @@ int queryDB(TAOS* taos, char* command) { } static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { - printf("tmq_commit_cb_print() commit %d\n", resp); + pError("tmq_commit_cb_print() commit %d\n", resp); } void build_consumer(SThreadInfo* pInfo) { @@ -318,7 +318,7 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { - printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); + pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); exit(-1); } @@ -375,7 +375,7 @@ void* consumeThreadFunc(void* param) { tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err) { - printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } @@ -388,14 +388,14 @@ void* consumeThreadFunc(void* param) { err = tmq_unsubscribe(pInfo->tmq); if (err) { - printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); pInfo->consumeMsgCnt = -1; return NULL; } err = tmq_consumer_close(pInfo->tmq); if (err) { - printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } pInfo->tmq = NULL; @@ -451,7 +451,7 @@ int32_t getConsumeInfo() { sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { - printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + pError("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); taosCloseFile(&g_fp); taos_free_result(pRes); diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 13f8cde3e3bc43635f92d2c52978a35fef519bbb..1639fd1ca681ab0bb980b2fd1fca8b34d58e15f3 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -332,7 +332,7 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shellInitArgs(argc, argv); shell.info.clientVersion = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; + "Copyright (c) 2022 by TAOS Data, Inc. All rights reserved.\n\n"; shell.info.promptHeader = "taos> "; shell.info.promptContinue = " -> "; shell.info.promptSize = 6; diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index f4f7c893c43166fe5d0ca03725122dc20b6665fe..ef71f3fce6f8b21f09f85ed373233cc5714e6316 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -53,79 +53,6 @@ static void shellResetCommand(SShellCmd *cmd, const char s[]); static void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos); static void shellShowOnScreen(SShellCmd *cmd); -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -// static void shellPrintContinuePrompt() { printf("%s", shell.args.promptContinue); } -// static void shellPrintPrompt() { printf("%s", shell.args.promptHeader); } - -void shellUpdateBuffer(SShellCmd *cmd) { - if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); - strcat(cmd->buffer, cmd->command); - - memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE); - cmd->cursorOffset = 0; -} - -int shellIsReadyGo(SShellCmd *cmd) { - char *total = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); - memset(total, 0, SHELL_MAX_COMMAND_SIZE); - sprintf(total, "%s%s", cmd->buffer, cmd->command); - - char *reg_str = - "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" - "\\s*clear\\s*$)"; - if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) { - taosMemoryFree(total); - return 1; - } - - taosMemoryFree(total); - return 0; -} - -void shellInsertChar(SShellCmd *cmd, char c) { - if (cmd->cursorOffset >= SHELL_MAX_COMMAND_SIZE) { - fprintf(stdout, "sql is larger than %d bytes", SHELL_MAX_COMMAND_SIZE); - return; - } - cmd->command[cmd->cursorOffset++] = c; -} - -int32_t shellReadCommand(char command[]) { - SShellCmd cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); - cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); - - // Read input. - char c; - while (1) { - c = getchar(); - - switch (c) { - case '\n': - case '\r': - if (shellIsReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosMemoryFree(cmd.buffer); - cmd.buffer = NULL; - taosMemoryFree(cmd.command); - cmd.command = NULL; - return 0; - } else { - // shellPrintContinuePrompt(); - shellUpdateBuffer(&cmd); - } - break; - default: - shellInsertChar(&cmd, c); - } - } - - return 0; -} - -#else - int32_t shellCountPrefixOnes(uint8_t c) { uint8_t mask = 127; mask = ~mask; @@ -181,7 +108,10 @@ void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { cmd->cursorOffset += size; cmd->screenOffset += taosWcharWidth(wc); cmd->endOffset += taosWcharWidth(wc); +#ifdef WINDOWS +#else shellShowOnScreen(cmd); +#endif } void shellBackspaceChar(SShellCmd *cmd) { @@ -371,17 +301,33 @@ void shellResetCommand(SShellCmd *cmd, const char s[]) { shellShowOnScreen(cmd); } -void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) { + +void shellGetScreenSize(int32_t *ws_col, int32_t *ws_row) { +#ifdef WINDOWS + CONSOLE_SCREEN_BUFFER_INFO csbi; + GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi); + if (ws_col != NULL) *ws_col = csbi.srWindow.Right - csbi.srWindow.Left + 1; + if (ws_row != NULL) *ws_row = csbi.srWindow.Bottom - csbi.srWindow.Top + 1; +#else struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); - w.ws_col = 120; - w.ws_row = 30; + if (ws_col != NULL) *ws_col = 120; + if (ws_row != NULL) *ws_row = 30; + } else { + if (ws_col != NULL) *ws_col = w.ws_col; + if (ws_row != NULL) *ws_row = w.ws_row; } +#endif +} + +void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) { + int32_t ws_col; + shellGetScreenSize(&ws_col, NULL); - int32_t cursor_x = cursor_pos / w.ws_col; - int32_t cursor_y = cursor_pos % w.ws_col; - int32_t command_x = ecmd_pos / w.ws_col; + int32_t cursor_x = cursor_pos / ws_col; + int32_t cursor_y = cursor_pos % ws_col; + int32_t command_x = ecmd_pos / ws_col; shellPositionCursor(cursor_y, LEFT); shellPositionCursor(command_x - cursor_x, DOWN); fprintf(stdout, "\033[2K"); @@ -393,12 +339,8 @@ void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) { } void shellShowOnScreen(SShellCmd *cmd) { - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - // fprintf(stderr, "No stream device\n"); - w.ws_col = 120; - w.ws_row = 30; - } + int32_t ws_col; + shellGetScreenSize(&ws_col, NULL); TdWchar wc; int32_t size = 0; @@ -411,8 +353,7 @@ void shellShowOnScreen(SShellCmd *cmd) { } else { sprintf(total_string, "%s%s", shell.info.promptContinue, cmd->command); } - - int32_t remain_column = w.ws_col; + int32_t remain_column = ws_col; for (char *str = total_string; size < cmd->commandSize + PSIZE;) { int32_t ret = taosMbToWchar(&wc, str, MB_CUR_MAX); if (ret < 0) break; @@ -425,10 +366,10 @@ void shellShowOnScreen(SShellCmd *cmd) { } else { if (remain_column == width) { printf("%lc\n\r", wc); - remain_column = w.ws_col; + remain_column = ws_col; } else { printf("\n\r%lc", wc); - remain_column = w.ws_col - width; + remain_column = ws_col - width; } } @@ -436,17 +377,16 @@ void shellShowOnScreen(SShellCmd *cmd) { } taosMemoryFree(total_string); - // Position the cursor int32_t cursor_pos = cmd->screenOffset + PSIZE; int32_t ecmd_pos = cmd->endOffset + PSIZE; - int32_t cursor_x = cursor_pos / w.ws_col; - int32_t cursor_y = cursor_pos % w.ws_col; - // int32_t cursor_y = cursor % w.ws_col; - int32_t command_x = ecmd_pos / w.ws_col; - int32_t command_y = ecmd_pos % w.ws_col; - // int32_t command_y = (command.size() + PSIZE) % w.ws_col; + int32_t cursor_x = cursor_pos / ws_col; + int32_t cursor_y = cursor_pos % ws_col; + // int32_t cursor_y = cursor % ws_col; + int32_t command_x = ecmd_pos / ws_col; + int32_t command_y = ecmd_pos % ws_col; + // int32_t command_y = (command.size() + PSIZE) % ws_col; shellPositionCursor(command_y, LEFT); shellPositionCursor(command_x, UP); shellPositionCursor(cursor_x, DOWN); @@ -490,7 +430,11 @@ int32_t shellReadCommand(char *command) { case 3: printf("\n"); shellResetCommand(&cmd, ""); - kill(0, SIGINT); + #ifdef WINDOWS + raise(SIGINT); + #else + kill(0, SIGINT); + #endif break; case 4: // EOF or Ctrl+D printf("\n"); @@ -503,7 +447,10 @@ int32_t shellReadCommand(char *command) { break; case '\n': case '\r': + #ifdef WINDOWS + #else printf("\n"); + #endif if (shellIsReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); taosMemoryFreeClear(cmd.buffer); @@ -608,5 +555,3 @@ int32_t shellReadCommand(char *command) { return 0; } - -#endif \ No newline at end of file diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 21fd3d03597e66ee2f8a2d1c13dee74d698bd605..39b97004ff5ea631f7f8c4ea7d049480283eeaba 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -29,11 +29,11 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD static int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); static void shellPrintNChar(const char *str, int32_t length, int32_t width); static void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision); -static int32_t shellVerticalPrintResult(TAOS_RES *tres); +static int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql); static int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision); static void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields); -static int32_t shellHorizontalPrintResult(TAOS_RES *tres); -static int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical); +static int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql); +static int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql); static void shellReadHistory(); static void shellWriteHistory(); static void shellPrintError(TAOS_RES *tres, int64_t st); @@ -121,7 +121,7 @@ int32_t shellRunCommand(char *command) { char quote = 0, *cmd = command; for (char c = *command++; c != 0; c = *command++) { if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) { - command ++; + command++; continue; } @@ -190,7 +190,7 @@ void shellRunSingleCommandImp(char *command) { if (pFields != NULL) { // select and show kinds of commands int32_t error_no = 0; - int32_t numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); + int32_t numOfRows = shellDumpResult(pSql, fname, &error_no, printMode, command); if (numOfRows < 0) return; et = taosGetTimestampUs(); @@ -272,6 +272,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i return; } + int n; char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -280,20 +281,37 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i case TSDB_DATA_TYPE_TINYINT: taosFprintfFile(pFile, "%d", *((int8_t *)val)); break; + case TSDB_DATA_TYPE_UTINYINT: + taosFprintfFile(pFile, "%u", *((uint8_t *)val)); + break; case TSDB_DATA_TYPE_SMALLINT: taosFprintfFile(pFile, "%d", *((int16_t *)val)); break; + case TSDB_DATA_TYPE_USMALLINT: + taosFprintfFile(pFile, "%u", *((uint16_t *)val)); + break; case TSDB_DATA_TYPE_INT: taosFprintfFile(pFile, "%d", *((int32_t *)val)); break; + case TSDB_DATA_TYPE_UINT: + taosFprintfFile(pFile, "%u", *((uint32_t *)val)); + break; case TSDB_DATA_TYPE_BIGINT: taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val)); break; + case TSDB_DATA_TYPE_UBIGINT: + taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val)); + break; case TSDB_DATA_TYPE_FLOAT: taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val)); + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, length)) { + taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); + } else { + taosFprintfFile(pFile, "%s", buf); + } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: @@ -435,6 +453,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t return; } + int n; char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -468,7 +487,12 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t printf("%*.5f", width, GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - printf("%*.9f", width, GET_DOUBLE_VAL(val)); + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, width)) { + printf("%*.15e", width, GET_DOUBLE_VAL(val)); + } else { + printf("%s", buf); + } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: @@ -483,7 +507,16 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t } } -int32_t shellVerticalPrintResult(TAOS_RES *tres) { +bool shellIsLimitQuery(const char *sql) { + //todo refactor + if (taosStrCaseStr(sql, " limit ") != NULL) { + return true; + } + + return false; +} + +int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; @@ -503,7 +536,7 @@ int32_t shellVerticalPrintResult(TAOS_RES *tres) { uint64_t resShowMaxNum = UINT64_MAX; - if (shell.args.commands == NULL && shell.args.file[0] == 0) { + if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsLimitQuery(sql)) { resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } @@ -525,8 +558,13 @@ int32_t shellVerticalPrintResult(TAOS_RES *tres) { putchar('\n'); } } else if (showMore) { - printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); - printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n"); + printf("\n"); + printf(" Notice: The result shows only the first %d rows.\n", SHELL_DEFAULT_RES_SHOW_NUM); + printf(" You can use the `LIMIT` clause to get fewer result to show.\n"); + printf(" Or use '>>' to redirect the whole set of the result to a specified file.\n"); + printf("\n"); + printf(" You can use Ctrl+C to stop the underway fetching.\n"); + printf("\n"); showMore = 0; } @@ -618,7 +656,7 @@ void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) { putchar('\n'); } -int32_t shellHorizontalPrintResult(TAOS_RES *tres) { +int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; @@ -637,7 +675,7 @@ int32_t shellHorizontalPrintResult(TAOS_RES *tres) { uint64_t resShowMaxNum = UINT64_MAX; - if (shell.args.commands == NULL && shell.args.file[0] == 0) { + if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsLimitQuery(sql)) { resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } @@ -655,8 +693,13 @@ int32_t shellHorizontalPrintResult(TAOS_RES *tres) { } putchar('\n'); } else if (showMore) { - printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); - printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n"); + printf("\n"); + printf(" Notice: The result shows only the first %d rows.\n", SHELL_DEFAULT_RES_SHOW_NUM); + printf(" You can use the `LIMIT` clause to get fewer result to show.\n"); + printf(" Or use '>>' to redirect the whole set of the result to a specified file.\n"); + printf("\n"); + printf(" You can use Ctrl+C to stop the underway fetching.\n"); + printf("\n"); showMore = 0; } @@ -667,14 +710,14 @@ int32_t shellHorizontalPrintResult(TAOS_RES *tres) { return numOfRows; } -int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical) { +int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql) { int32_t numOfRows = 0; if (fname != NULL) { numOfRows = shellDumpResultToFile(fname, tres); } else if (vertical) { - numOfRows = shellVerticalPrintResult(tres); + numOfRows = shellVerticalPrintResult(tres, sql); } else { - numOfRows = shellHorizontalPrintResult(tres); + numOfRows = shellHorizontalPrintResult(tres, sql); } *error_no = taos_errno(tres); diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index 345b85d896c92c3b28a0e6cc78f9ce775a03bd4a..3355c20109fbfabbbf3ce50184235829f0cb1eb7 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -119,7 +119,7 @@ static void shellWorkAsServer() { memcpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn)); rpcInit.localPort = pArgs->port; rpcInit.label = "CHK"; - rpcInit.numOfThreads = tsNumOfRpcThreads; + rpcInit.numOfThreads = 2; rpcInit.cfp = (RpcCfp)shellProcessMsg; rpcInit.sessions = 10; rpcInit.connType = TAOS_CONN_SERVER; diff --git a/tools/taos-tools b/tools/taos-tools index 0ae9f872c26d5da8cb61aa9eb00b5c7aeba10ec4..0aad27d725f4ee6b18daf1db0c07d933aed16eea 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 0ae9f872c26d5da8cb61aa9eb00b5c7aeba10ec4 +Subproject commit 0aad27d725f4ee6b18daf1db0c07d933aed16eea