diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index fc9a51af654a9a3cecc57046048d3976e5ef6bc5..0000000000000000000000000000000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile - -# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 -ARG VARIANT="bullseye" -FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} - -# [Optional] Uncomment this section to install additional packages. -# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 9b752d091dee2b91e9cde8a2a6b825054901434a..0000000000000000000000000000000000000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,32 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp -{ - "name": "C++", - "build": { - "dockerfile": "Dockerfile", - // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 - // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon - "args": { "VARIANT": "ubuntu-21.04" } - }, - "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], - - // Set *default* container specific settings.json values on container create. - "settings": {}, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-vscode.cpptools", - "ms-vscode.cmake-tools", - "austin.code-gnu-global", - "visualstudioexptteam.vscodeintel" - ], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "gcc -v", - - // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" -} diff --git a/include/client/taos.h b/include/client/taos.h index 0f7edc9fedba746c1f5510063b0acc4bd8dea95b..19d191b84e7db8ed42430b9f82440b2b02075f06 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -73,10 +73,12 @@ typedef struct taosField { #define DLL_EXPORT #endif -DLL_EXPORT int taos_init(); +typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); + DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); @@ -154,14 +156,14 @@ DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); // TODO: the return value should be `const` -DLL_EXPORT char *taos_get_server_info(TAOS *taos); -DLL_EXPORT char *taos_get_client_info(); -DLL_EXPORT char *taos_errstr(TAOS_RES *tres); +DLL_EXPORT const char *taos_get_server_info(TAOS *taos); +DLL_EXPORT const char *taos_get_client_info(); +DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); DLL_EXPORT int taos_errno(TAOS_RES *tres); -DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param); -DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); +DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); +DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 6f25d0fb8706c12a152ec1bfad00ca57a838713b..a75c4a7bac9555af40f667442eb45c12974c5e88 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -77,7 +77,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STB_VGROUP, "stb-vgroup" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) @@ -214,6 +214,11 @@ typedef enum _mgmt_table { extern char *taosMsg[]; +typedef struct SBuildTableMetaInput { + int32_t vgId; + char *tableFullName; +} SBuildTableMetaInput; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -358,6 +363,7 @@ typedef struct { int32_t pid; char app[TSDB_APP_NAME_LEN]; char db[TSDB_DB_NAME_LEN]; + int64_t startTime; } SConnectMsg; typedef struct SEpSet { @@ -368,19 +374,17 @@ typedef struct SEpSet { } SEpSet; typedef struct { - int32_t acctId; - int32_t clusterId; - int32_t connId; - int8_t superAuth; - int8_t readAuth; - int8_t writeAuth; - int8_t reserved[5]; - SEpSet epSet; + int32_t acctId; + uint32_t clusterId; + int32_t connId; + int8_t superUser; + int8_t reserved[5]; + SEpSet epSet; } SConnectRsp; typedef struct { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; int32_t maxUsers; int32_t maxDbs; int32_t maxTimeSeries; @@ -395,7 +399,7 @@ typedef struct { typedef struct { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; } SCreateUserMsg, SAlterUserMsg; typedef struct { @@ -774,9 +778,8 @@ typedef struct { } SStbInfoMsg; typedef struct { + SMsgHead msgHead; char tableFname[TSDB_TABLE_FNAME_LEN]; - int8_t createFlag; - char tags[]; } STableInfoMsg; typedef struct { @@ -791,6 +794,20 @@ typedef struct SSTableVgroupMsg { int32_t numOfTables; } SSTableVgroupMsg, SSTableVgroupRspMsg; +typedef struct SVgroupInfo { + int32_t vgId; + int8_t numOfEps; + SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; +} SVgroupInfo; + +typedef struct SVgroupListRspMsg { + int32_t vgroupNum; + int32_t vgroupVersion; + SVgroupInfo vgroupInfo[]; +} SVgroupListRspMsg; + +typedef SVgroupListRspMsg SVgroupListInfo; + typedef struct { int32_t vgId; int8_t numOfEps; @@ -961,8 +978,8 @@ typedef struct { char user[TSDB_USER_LEN]; char spi; char encrypt; - char secret[TSDB_KEY_LEN]; - char ckey[TSDB_KEY_LEN]; + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; } SAuthMsg, SAuthRsp; typedef struct { diff --git a/include/common/tep.h b/include/common/tep.h new file mode 100644 index 0000000000000000000000000000000000000000..2addc11e5a6251be8cfcb1ca4ef688edcfbde56f --- /dev/null +++ b/include/common/tep.h @@ -0,0 +1,17 @@ +#ifndef TDENGINE_TEP_H +#define TDENGINE_TEP_H + +#include "os.h" +#include "taosmsg.h" + +typedef struct SCorEpSet { + int32_t version; + SEpSet epSet; +} SCorEpSet; + +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); + +void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); + +#endif // TDENGINE_TEP_H diff --git a/include/common/tglobal.h b/include/common/tglobal.h index aedf4122bb5cbfb83bbff56c530198f870db176b..a9b0e607614811281a63f62349b49b91ed15cb5d 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -81,8 +81,6 @@ extern int64_t tsMaxRetentWindow; // db parameters in client extern int32_t tsCacheBlockSize; extern int32_t tsBlocksPerVnode; -extern int32_t tsMinTablePerVnode; -extern int32_t tsMaxTablePerVnode; extern int32_t tsTableIncStepPerVnode; extern int32_t tsMaxVgroupsPerDb; extern int16_t tsDaysPerFile; @@ -113,16 +111,8 @@ extern int8_t tsEnableSlaveQuery; extern int8_t tsEnableAdjustMaster; // restful -extern int8_t tsEnableHttpModule; extern int32_t tsRestRowLimit; -extern uint16_t tsHttpPort; -extern int32_t tsHttpCacheSessions; -extern int32_t tsHttpSessionExpire; -extern int32_t tsHttpMaxThreads; -extern int8_t tsHttpEnableCompress; -extern int8_t tsHttpEnableRecordSql; extern int8_t tsTelegrafUseFieldNum; -extern int8_t tsHttpDbNameMandatory; // mqtt extern int8_t tsEnableMqttModule; @@ -144,7 +134,6 @@ extern int8_t tsEnableStream; // internal extern int8_t tsPrintAuth; -extern int8_t tscEmbedded; extern char tsVnodeDir[]; extern char tsMnodeDir[]; extern int64_t tsTickPerDay[3]; @@ -193,7 +182,7 @@ extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) void taosInitGlobalCfg(); -int32_t taosCheckGlobalCfg(); +int32_t taosCheckAndPrintCfg(); int32_t taosCfgDynamicOptions(char *msg); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); diff --git a/include/common/tmessage.h b/include/common/tmessage.h new file mode 100644 index 0000000000000000000000000000000000000000..c728ee026eb05e111e5da67656763be2b128306c --- /dev/null +++ b/include/common/tmessage.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_TMESSAGE_H_ +#define _TD_COMMON_TMESSAGE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +extern int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); +extern int32_t (*tscProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); + + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_TMESSAGE_H_*/ diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 050b9c904fdf05bab6f25e9a8b93c8dfb0356846..5019191fd195fb34ec8ff03bbbf5469dd81aa24e 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -30,16 +30,23 @@ extern "C" { struct SCatalog; -typedef struct SMetaReq { - char clusterId[TSDB_CLUSTER_ID_LEN]; +typedef struct SDBVgroupInfo { + int32_t vgroupVersion; + SArray *vgId; + int32_t hashRange; + int32_t hashNum; +} SDBVgroupInfo; + +typedef struct SCatalogReq { + char clusterId[TSDB_CLUSTER_ID_LEN]; //???? SArray *pTableName; // table full name SArray *pUdf; // udf name - bool qNodeEpset; // valid qnode -} SMetaReq; + bool qNodeRequired; // valid qnode +} SCatalogReq; typedef struct SMetaData { - SArray *pTableMeta; // tableMeta - SArray *pVgroupInfo; // vgroupInfo list + SArray *pTableMeta; // STableMeta array + SArray *pVgroupInfo; // SVgroupInfo list SArray *pUdfList; // udf info list SEpSet *pEpSet; // qnode epset list } SMetaData; @@ -78,32 +85,71 @@ typedef struct STableMeta { SSchema schema[]; } STableMeta; +typedef struct SCatalogCfg { + +} SCatalogCfg; + +int32_t catalogInit(SCatalogCfg *cfg); + /** * Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side. * There is ONLY one SCatalog object for one process space, and this function returns a singleton. - * @param pMgmtEps + * @param clusterId + * @return + */ +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); + + + +int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); +int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList); +int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); + + + +int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo); +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); + + +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta); +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, STableMeta* pNewTableMeta); + + +/** + * get table's vgroup list. + * @param clusterId + * @pVgroupList - array of SVgroupInfo * @return */ -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); +int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, SArray* pVgroupList); + /** * Get the required meta data from mnode. * Note that this is a synchronized API and is also thread-safety. * @param pCatalog + * @param pMgmtEps * @param pMetaReq * @param pMetaData * @return */ -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); + + +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); + + /** - * Destroy catalog service handle + * Destroy catalog and relase all resources * @param pCatalog */ -void destroyCatalog(struct SCatalog* pCatalog); +void catalogDestroy(void); #ifdef __cplusplus } #endif -#endif /*_TD_CATALOG_H_*/ \ No newline at end of file +#endif /*_TD_CATALOG_H_*/ diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 2a8a7aad4b05abc1c80edde9eee706e38607450e..2afd9d48b76839682764dab87fb1dd7adc7796be 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -29,11 +29,6 @@ extern "C" { extern int tsRpcHeadSize; -typedef struct SRpcCorEpSet { - int32_t version; - SEpSet epSet; -} SRpcCorEpSet; - typedef struct SRpcConnInfo { uint32_t clientIp; uint16_t clientPort; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index ae1e630c6fff6caa46e897dec51207fe4ce3386c..744275e6ff66937ef0a61068bdb7e5aadf61f7da 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -32,6 +32,23 @@ extern int32_t wDebugFlag; #define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} #define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define WAL_PREFIX "wal" +#define WAL_PREFIX_LEN 3 +#define WAL_NOSUFFIX_LEN 20 +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) +#define WAL_LOG_SUFFIX "log" +#define WAL_INDEX_SUFFIX "idx" +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) + +#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) +#define WAL_CUR_POS_WRITABLE 1 +#define WAL_CUR_FILE_WRITABLE 2 +#define WAL_CUR_FAILED 4 + +#pragma pack(push,1) typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, @@ -43,6 +60,7 @@ typedef struct SWalReadHead { uint8_t msgType; int8_t reserved[2]; int32_t len; + //int64_t ingestTs; //not implemented int64_t version; char body[]; } SWalReadHead; @@ -71,25 +89,6 @@ typedef struct { SWalReadHead head; } SWalHead; -#define WAL_PREFIX "wal" -#define WAL_PREFIX_LEN 3 -#define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) -#define WAL_LOG_SUFFIX "log" -#define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) -#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFEUL)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -//#define WAL_FILE_NUM 1 // 3 -#define WAL_FILESET_MAX 128 - -#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) -#define WAL_CUR_POS_WRITABLE 1 -#define WAL_CUR_FILE_WRITABLE 2 -#define WAL_CUR_FAILED 4 - typedef struct SWalVer { int64_t firstVer; int64_t verInSnapshotting; @@ -101,24 +100,18 @@ typedef struct SWalVer { typedef struct SWal { // cfg SWalCfg cfg; - //total size - int64_t totSize; - //fsync seq - int32_t fsyncSeq; - //reference - int64_t refId; - //write tfd - int64_t writeLogTfd; - int64_t writeIdxTfd; - //wal lifecycle SWalVer vers; - //roll status - int64_t lastRollSeq; //file set int32_t writeCur; + int64_t writeLogTfd; + int64_t writeIdxTfd; SArray* fileInfoSet; //ctl int32_t curStatus; + int32_t fsyncSeq; + int64_t totSize; + int64_t refId; + int64_t lastRollSeq; pthread_mutex_t mutex; //path char path[WAL_PATH_LEN]; @@ -134,8 +127,9 @@ typedef struct SWalReadHandle { int64_t curVersion; int64_t capacity; int64_t status; //if cursor valid - SWalHead head; + SWalHead* pHead; } SWalReadHandle; +#pragma pack(pop) typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 86a9f10819f11d54cd860bfbc01ebb6ba9d34129..78112fc7a0f58d0659cdbd0043c47e814ca69ca0 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -57,7 +57,7 @@ int64_t taosGetPthreadId(pthread_t thread); void taosResetPthread(pthread_t* thread); bool taosComparePthread(pthread_t first, pthread_t second); int32_t taosGetPId(); -int32_t taosGetCurrentAPPName(char* name, int32_t* len); +int32_t taosGetAppName(char* name, int32_t* len); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 64a1fa84ad860c7ccb38332e6220ce2204c6a425..8400031f9bdfcf0979725f85261d93426d4d502c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -117,6 +117,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format") #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type") #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") +#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) @@ -501,6 +502,13 @@ int32_t* taosGetErrno(); // monitor #define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") +// catalog +#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) //catalog interval error +#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) //invalid catalog input parameters +#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) //catalog is not ready +#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error +#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) //catalog system error + #ifdef __cplusplus } #endif diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 15100423d5e4edac40a61f7436822cbeb1853aab..7dee01247b9e081585ab1ba4c4a1f907dec9cbc6 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 123 +#define TSDB_CFG_MAX_NUM 115 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 @@ -83,11 +83,11 @@ extern int32_t tsGlobalConfigNum; extern char * tsCfgStatusStr[]; void taosReadGlobalLogCfg(); -int32_t taosReadGlobalCfg(); -void taosPrintGlobalCfg(); +int32_t taosReadCfgFromFile(); +void taosPrintCfg(); void taosDumpGlobalCfg(); -void taosInitConfigOption(SGlobalCfg cfg); +void taosAddConfigOption(SGlobalCfg cfg); SGlobalCfg *taosGetConfigOption(const char *option); #ifdef __cplusplus diff --git a/include/util/tdef.h b/include/util/tdef.h index 28a1e24581d4a2086c9ec0f50933b811f8fc4b32..300b190e97a9093298ae849664f688975fcec7fe 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -161,7 +161,7 @@ do { \ #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 33 +#define TSDB_DB_NAME_LEN 65 #define TSDB_FULL_DB_NAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN) #define TSDB_FUNC_NAME_LEN 65 @@ -193,7 +193,7 @@ do { \ #define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_AUTH_LEN 16 -#define TSDB_KEY_LEN 16 +#define TSDB_PASSWORD_LEN 32 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 diff --git a/include/util/tlog.h b/include/util/tlog.h index 2ee60e43244017f4035aa850ff0b1de11f5ed9aa..9b6033e7fe550bff0c34dadf42e8d139aa703fe9 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -45,6 +45,8 @@ extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; +extern int32_t ctgDebugFlag; + #define DEBUG_FATAL 1U #define DEBUG_ERROR DEBUG_FATAL diff --git a/include/util/tutil.h b/include/util/tutil.h index 3a07b898c0e514f890fe05bd91a10cc5c7144926..c315948702af31b5dbcf8cde71331f132554532f 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -45,14 +45,25 @@ char *taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); void taosIp2String(uint32_t ip, char *str); void taosIpPort2String(uint32_t ip, uint16_t port, char *str); -int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; tMD5Init(&context); tMD5Update(&context, inBuf, (unsigned int)inLen); tMD5Final(&context); - memcpy(target, context.digest, TSDB_KEY_LEN); + memcpy(target, context.digest, tListLen(context.digest)); +} + +static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) { + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, inBuf, (unsigned int)len); + tMD5Final(&context); + + sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], + context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], + context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], + context.digest[13], context.digest[14], context.digest[15]); } #ifdef __cplusplus diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index bc0d439407f9e77f9e54de716e32663e24e5b767..b46afa31b9299cd3a41e68913181e03389d2a63c 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -2,9 +2,14 @@ aux_source_directory(src CLIENT_SRC) add_library(taos ${CLIENT_SRC}) target_include_directories( taos - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_SOURCE_DIR}/include/client" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( taos + PRIVATE common INTERFACE api + PRIVATE os util common transport parser ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h new file mode 100644 index 0000000000000000000000000000000000000000..3180923aff03a8d99d470f38e4d000c9274fe878 --- /dev/null +++ b/source/client/inc/clientInt.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLIENTINT_H +#define TDENGINE_CLIENTINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taos.h" +#include "taosmsg.h" +#include "thash.h" +#include "tlist.h" +#include "trpc.h" +#include "tdef.h" +#include "tmsgtype.h" +#include "tep.h" + +typedef struct SQueryExecMetric { + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server +} SQueryExecMetric; + +typedef struct SInstanceActivity { + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. + + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj +} SInstanceActivity; + +typedef struct SHeartBeatInfo { + void *pTimer; // timer, used to send request msg to mnode +} SHeartBeatInfo; + +typedef struct SAppInstInfo { + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceActivity summary; + SList *pConnList; // STscObj linked list + uint32_t clusterId; + void *pTransporter; +} SAppInstInfo; + +typedef struct SAppInfo { + int64_t startTime; + char appName[TSDB_APP_NAME_LEN]; + char *ep; + int32_t pid; + int32_t numOfThreads; + SHeartBeatInfo hb; + SHashObj *pInstMap; +} SAppInfo; + +typedef struct STscObj { + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char acctId[TSDB_ACCT_ID_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + uint32_t connId; + uint64_t id; // ref ID returned by taosAddRef +// struct SSqlObj *sqlList; + void *pTransporter; + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj from this tscObj + SAppInstInfo *pAppInfo; +} STscObj; + +typedef struct SReqBody { + tsem_t rspSem; // not used now + void* fp; + void* param; +} SRequestBody; + +typedef struct SRequestObj { + uint64_t requestId; + int32_t type; // request type + STscObj *pTscObj; + SQueryExecMetric metric; + char *sqlstr; // sql string + SRequestBody body; + int64_t self; + char *msgBuf; + int32_t code; + void *pInfo; // sql parse info, generated by parser module +} SRequestObj; + +typedef struct SRequestMsgBody { + int32_t msgType; + void *pData; + int32_t msgLen; + uint64_t requestId; + uint64_t requestObjRefId; +} SRequestMsgBody; + +extern SAppInfo appInfo; +extern int32_t tscReqRef; +extern void *tscQhandle; +extern int32_t tscConnRef; + +extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg); +extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); + +int taos_init(); + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo); +void destroyTscObj(void*pObj); + +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void destroyRequest(SRequestObj* pRequest); + +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); + +void taos_init_imp(void); +int taos_options_imp(TSDB_OPTION option, const char *str); + +void* openTransporter(const char *user, const char *auth); + +void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); +void initMsgHandleFp(); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTINT_H diff --git a/source/client/inc/tscLog.h b/source/client/inc/tscLog.h new file mode 100644 index 0000000000000000000000000000000000000000..f205a50227308e71e27e9715e9f819078be5a20a --- /dev/null +++ b/source/client/inc/tscLog.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSCLOG_H +#define TDENGINE_TSCLOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/client/src/client.c b/source/client/src/client.c index b1663239e6d6d66f154ee0167731d7b0815db34f..863e96aa744ee42f0b15fb0f4376bd4351b17220 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -13,11 +13,51 @@ * along with this program. If not, see . */ -//#include "taos.h" +#include "os.h" +#include "tdef.h" +#include "tglobal.h" +#include "clientInt.h" +#include "tscLog.h" -//TAOS_RES *taos_query(TAOS *taos, const char *sql) { -// -//} +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { + int32_t p = (port != 0)? port:tsServerPort; + + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (pass == NULL) { + pass = TSDB_DEFAULT_PASS; + } + + return taos_connect_internal(ip, user, pass, NULL, db, p); +} + +TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { + tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (auth == NULL) { + tscError("No auth info is given, failed to connect to server"); + return NULL; + } + + return taos_connect_internal(ip, user, NULL, auth, db, port); +} + +TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { + char ipStr[TSDB_EP_LEN] = {0}; + char dbStr[TSDB_DB_NAME_LEN] = {0}; + char userStr[TSDB_USER_LEN] = {0}; + char passStr[TSDB_PASSWORD_LEN] = {0}; + + strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + return taos_connect(ipStr, userStr, passStr, dbStr, port); +} -int taos_init() { return 0; } -void taos_cleanup(void) {} diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c new file mode 100644 index 0000000000000000000000000000000000000000..2ae79f1947e9385ca5bb31cddc01be803e4d6d0d --- /dev/null +++ b/source/client/src/clientImpl.c @@ -0,0 +1,283 @@ +#include +#include "clientInt.h" +#include "tdef.h" +#include "tep.h" +#include "tglobal.h" +#include "tmsgtype.h" +#include "tref.h" +#include "tscLog.h" + +static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); +static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody); +static void destroyConnectMsg(SRequestMsgBody* pMsgBody); + +static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId); + +static bool stringLengthCheck(const char* str, size_t maxsize) { + if (str == NULL) { + return false; + } + + size_t len = strlen(str); + if (len <= 0 || len > maxsize) { + return false; + } + + return true; +} + +static bool validateUserName(const char* user) { + return stringLengthCheck(user, TSDB_USER_LEN - 1); +} + +static bool validatePassword(const char* passwd) { + return stringLengthCheck(passwd, TSDB_PASSWORD_LEN - 1); +} + +static bool validateDbName(const char* db) { + return stringLengthCheck(db, TSDB_DB_NAME_LEN - 1); +} + +static char* getClusterKey(const char* user, const char* auth, const char* ip, int32_t port) { + char key[512] = {0}; + snprintf(key, sizeof(key), "%s:%s:%s:%d", user, auth, ip, port); + return strdup(key); +} + +static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); + +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { + if (taos_init() != TSDB_CODE_SUCCESS) { + return NULL; + } + + if (!validateUserName(user)) { + terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; + return NULL; + } + + char tmp[TSDB_DB_NAME_LEN] = {0}; + if (db != NULL) { + if(!validateDbName(db)) { + terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; + return NULL; + } + + tstrncpy(tmp, db, sizeof(tmp)); + strdequote(tmp); + } + + char secretEncrypt[32] = {0}; + if (auth == NULL) { + if (!validatePassword(pass)) { + terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; + return NULL; + } + + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); + } else { + tstrncpy(secretEncrypt, auth, tListLen(secretEncrypt)); + } + + SCorEpSet epSet = {0}; + if (ip) { + if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { + return NULL; + } + + if (port) { + epSet.epSet.port[0] = port; + } + } else { + if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { + return NULL; + } + } + + char* key = getClusterKey(user, secretEncrypt, ip, port); + + SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + if (pInst == NULL) { + pInst = calloc(1, sizeof(struct SAppInstInfo)); + + pInst->mgmtEp = epSet; + pInst->pTransporter = openTransporter(user, secretEncrypt); + + taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES); + } + + return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst); +} + +int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { + pEpSet->version = 0; + + // init mgmt ip set + SEpSet *mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; + + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(firstEp, mgmtEpSet->fqdn[0], &(mgmtEpSet->port[0])); + mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(secondEp, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); + mgmtEpSet->numOfEps++; + } + + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + return 0; +} + +STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { + STscObj *pTscObj = createTscObj(user, auth, ip, port, pAppInfo); + if (NULL == pTscObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pTscObj; + } + + SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT); + if (pRequest == NULL) { + destroyTscObj(pTscObj); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SRequestMsgBody body = {0}; + buildConnectMsg(pRequest, &body); + + int64_t transporterId = 0; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyConnectMsg(&body); + + if (pRequest->code != TSDB_CODE_SUCCESS) { + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); + printf("failed to connect to server, reason: %s\n\n", errorMsg); + + destroyRequest(pRequest); + taos_close(pTscObj); + pTscObj = NULL; + } else { + tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter); + destroyRequest(pRequest); + } + + return pTscObj; +} + +static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; + pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); + if (pConnect == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + // TODO refactor full_name + char *db; // ugly code to move the space + + STscObj *pObj = pRequest->pTscObj; + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + pConnect->pid = htonl(appInfo.pid); + pConnect->startTime = htobe64(appInfo.startTime); + tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); + + pMsgBody->pData = pConnect; + return 0; +} + +static void destroyConnectMsg(SRequestMsgBody* pMsgBody) { + assert(pMsgBody != NULL); + tfree(pMsgBody->pData); +} + +int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) { + char *pMsg = rpcMallocCont(pBody->msgLen); + if (NULL == pMsg) { + tscError("0x%"PRIx64" msg:%s malloc failed", pBody->requestId, taosMsg[pBody->msgType]); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + memcpy(pMsg, pBody->pData, pBody->msgLen); + SRpcMsg rpcMsg = { + .msgType = pBody->msgType, + .pCont = pMsg, + .contLen = pBody->msgLen, + .ahandle = (void*) pBody->requestObjRefId, + .handle = NULL, + .code = 0 + }; + + rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId); + return TSDB_CODE_SUCCESS; +} + +void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + int64_t requestRefId = (int64_t)pMsg->ahandle; + + SRequestObj *pRequest = (SRequestObj *)taosAcquireRef(tscReqRef, requestRefId); + if (pRequest == NULL) { + rpcFreeCont(pMsg->pCont); + return; + } + + assert(pRequest->self == requestRefId); + pRequest->metric.rsp = taosGetTimestampMs(); + + pRequest->code = pMsg->code; + + STscObj *pTscObj = pRequest->pTscObj; + if (pEpSet) { + if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, pEpSet)) { + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, pEpSet); + } + } + + /* + * There is not response callback function for submit response. + * The actual inserted number of points is the first number. + */ + if (pMsg->code == TSDB_CODE_SUCCESS) { + tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); + if (handleRequestRspFp[pRequest->type]) { + pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen); + } + } else { + tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen); + } + + taosReleaseRef(tscReqRef, requestRefId); + rpcFreeCont(pMsg->pCont); + + sem_post(&pRequest->body.rspSem); +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c new file mode 100644 index 0000000000000000000000000000000000000000..62466a096df95950261ddd89d9caed7e3524870c --- /dev/null +++ b/source/client/src/clientMsgHandler.c @@ -0,0 +1,3212 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "clientInt.h" +#include "tmsgtype.h" +#include "trpc.h" +#include "tscLog.h" + +int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; +int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); + +#if 0 +void tscProcessActivityTimer(void *handle, void *tmrId); + +static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); + +static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static int32_t getWaitingTimeInterval(int32_t count) { + int32_t initial = 100; // 100 ms by default + if (count <= 1) { + return 0; + } + + return initial * ((2u)<<(count - 2)); +} + +static int32_t vgIdCompare(const void *lhs, const void *rhs) { + int32_t left = *(int32_t *)lhs; + int32_t right = *(int32_t *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} +static int32_t removeDupVgid(int32_t *src, int32_t sz) { + if (src == NULL || sz <= 0) { + return 0; + } + qsort(src, sz, sizeof(src[0]), vgIdCompare); + + int32_t ret = 1; + for (int i = 1; i < sz; i++) { + if (src[i] != src[i - 1]) { + src[ret++] = src[i]; + } + } + return ret; +} + +static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupMsg* pVgroupInfo) { + assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); + + // Issue the query to one of the vnode among a vgroup randomly. + // change the inUse property would not affect the isUse attribute of STableMeta + pEpSet->inUse = rand() % pVgroupInfo->numOfEps; + + // apply the FQDN string length check here + bool existed = false; + + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + + int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + if (len > 0) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + existed = true; + } + } + + assert(existed); +} + +static void tscDumpMgmtEpSet(SSqlObj *pSql) { + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + pSql->epSet = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); +} +static void tscEpSetHtons(SRpcEpSet *s) { + for (int32_t i = 0; i < s->numOfEps; i++) { + s->port[i] = htons(s->port[i]); + } +} + +bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { + return false; + } + + for (int32_t i = 0; i < s1->numOfEps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} + +void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { + // no need to update if equal + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginWrite(&pCorEpSet->version); + pCorEpSet->epSet = *pEpSet; + taosCorEndWrite(&pCorEpSet->version); +} + +static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { + if (pVgroupInfo == NULL) { return;} + int8_t inUse = pVgroupInfo->inUse; + pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); + pEpSet->port[i] = pVgroupInfo->ep[i].port; + } +} + +static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { + SSqlCmd *pCmd = &pSql->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { + return; + } + + int32_t vgId = -1; + if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { + vgId = extractSTableQueryVgroupId(pTableMetaInfo); + } else { + vgId = pTableMetaInfo->pTableMeta->vgId; + } + + assert(vgId > 0); + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); + assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); + + tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + vgroupInfo.inUse = pEpSet->inUse; + vgroupInfo.numOfEps = pEpSet->numOfEps; + for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { + tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + vgroupInfo.ep[i].port = pEpSet->port[i]; + } + + tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); + + // Update the local cached epSet info cached by SqlObj + int32_t inUse = pSql->epSet.inUse; + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + tscDebug("0x%"PRIx64" update the epSet in SqlObj, in use before:%d, after:%d", pSql->self, inUse, pSql->epSet.inUse); + +} + +int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { + assert(pTableMetaInfo != NULL); + + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + return vgId; +} + +void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { + STscObj *pObj = (STscObj *)param; + if (pObj == NULL) return; + + if (pObj != pObj->signature) { + tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); + return; + } + + SSqlObj *pSql = tres; + SSqlRes *pRes = &pSql->res; + + if (code == TSDB_CODE_SUCCESS) { + SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; + SRpcEpSet *epSet = &pRsp->epSet; + if (epSet->numOfEps > 0) { + tscEpSetHtons(epSet); + + //SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { + // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); + // for (int8_t i = 0; i < epSet->numOfEps; i++) { + // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); + // } + //} + //concurrency problem, update mgmt epset anyway + tscUpdateMgmtEpSet(pSql, epSet); + } + + pSql->pTscObj->connId = htonl(pRsp->connId); + + if (pRsp->killConnection) { + tscKillConnection(pObj); + return; + } else { + if (pRsp->queryId) { + tscKillQuery(pObj, htonl(pRsp->queryId)); + } + + if (pRsp->streamId) { + tscKillStream(pObj, htonl(pRsp->streamId)); + } + } + + int32_t total = htonl(pRsp->totalDnodes); + int32_t online = htonl(pRsp->onlineDnodes); + assert(online <= total); + + if (online < total) { + tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); + pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[0] = total; + pRes->length[1] = online; + } else { + tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[1] = 0; + if (pRes->length[0] == 0) { + pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node + } + } + + if (pObj->hbrid != 0) { + int32_t waitingDuring = tsShellActivityTimer * 500; + tscDebug("0x%"PRIx64" send heartbeat in %dms", pSql->self, waitingDuring); + + taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); + } else { + tscDebug("0x%"PRIx64" start to close tscObj:%p, not send heartbeat again", pSql->self, pObj); + } +} + +void tscProcessActivityTimer(void *handle, void *tmrId) { + int64_t rid = (int64_t) handle; + STscObj *pObj = taosAcquireRef(tscRefId, rid); + if (pObj == NULL) { + return; + } + + SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHB == NULL) { + taosReleaseRef(tscRefId, rid); + return; + } + + assert(pHB->self == pObj->hbrid); + + pHB->retry = 0; + int32_t code = tscBuildAndSendRequest(pHB, NULL); + taosReleaseRef(tscObjRef, pObj->hbrid); + + if (code != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); + } + + taosReleaseRef(tscRefId, rid); +} + +int tscSendMsgToServer(SSqlObj *pSql) { + STscObj* pObj = pSql->pTscObj; + SSqlCmd* pCmd = &pSql->cmd; + + char *pMsg = rpcMallocCont(pCmd->payloadLen); + if (NULL == pMsg) { + tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + // set the mgmt ip list + if (pSql->cmd.command >= TSDB_SQL_MGMT) { + tscDumpMgmtEpSet(pSql); + } + + memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); + + SRpcMsg rpcMsg = { + .msgType = pSql->cmd.msgType, + .pCont = pMsg, + .contLen = pSql->cmd.payloadLen, + .ahandle = (void*)pSql->self, + .handle = NULL, + .code = 0 + }; + + rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); + return TSDB_CODE_SUCCESS; +} + +// handle three situation +// 1. epset retry, only return last failure ep +// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn +// 3. other situation, no expected +void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + char* msgBuf = tscGetErrorMsgPayload(pCmd); + + if (pEpSet) { + sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); + } else if (pCmd->command >= TSDB_SQL_MGMT) { + SRpcEpSet tEpset; + + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + tEpset = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); + + sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); + } else { + sprintf(msgBuf, "%s", tstrerror(pRes->code)); + } +} + +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { + TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); + if (pSql == NULL) { + rpcFreeCont(rpcMsg->pCont); + return; + } + + assert(pSql->self == handle); + + STscObj *pObj = pSql->pTscObj; + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pSql->rpcRid = -1; + if (pObj->signature != pObj) { + tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); + + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { + tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", + pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); + + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + + if (pEpSet) { + if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { + if (pCmd->command < TSDB_SQL_MGMT) { + tscUpdateVgroupInfo(pSql, pEpSet); + } else { + tscUpdateMgmtEpSet(pSql, pEpSet); + } + } + } + + int32_t cmd = pCmd->command; + + // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema + if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + pSql->cmd.insertParam.schemaAttached = 1; + } + + // single table query error need to be handled here. + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || + rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { + + // 1. super table subquery + // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer + if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | + TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || + (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { + // do nothing in case of super table subquery + } else { + pSql->retry += 1; + tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); + + pSql->res.code = rpcMsg->code; // keep the previous error code + if (pSql->retry > pSql->maxRetry) { + tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); + } else { + // wait for a little bit moment and then retry + // todo do not sleep in rpc callback thread, add this process into queue to process + if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + int32_t duration = getWaitingTimeInterval(pSql->retry); + taosMsleep(duration); + } + + pSql->retryReason = rpcMsg->code; + rpcMsg->code = tscRenewTableMeta(pSql, 0); + // if there is an error occurring, proceed to the following error handling procedure. + if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + } + } + } + + pRes->rspLen = 0; + + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { + tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; + } + + if (pRes->code == TSDB_CODE_SUCCESS) { + tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); + pSql->retry = 0; + } + + if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { + assert(rpcMsg->msgType == pCmd->msgType + 1); + pRes->code = rpcMsg->code; + pRes->rspType = rpcMsg->msgType; + pRes->rspLen = rpcMsg->contLen; + + if (pRes->rspLen > 0 && rpcMsg->pCont) { + char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (tmp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + } else { + pRes->pRsp = tmp; + memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); + } + } else { + tfree(pRes->pRsp); + } + + /* + * There is not response callback function for submit response. + * The actual inserted number of points is the first number. + */ + if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) { + SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; + pMsg->code = htonl(pMsg->code); + pMsg->numOfRows = htonl(pMsg->numOfRows); + pMsg->affectedRows = htonl(pMsg->affectedRows); + pMsg->failedRows = htonl(pMsg->failedRows); + pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); + + pRes->numOfRows += pMsg->affectedRows; + tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], + tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); + } else { + tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); + } + } + + if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { + rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); + } + + bool shouldFree = tscShouldBeFreed(pSql); + if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + pRes->code = rpcMsg->code; + } + + rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; + if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) { + tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); + tscSetFqdnErrorMsg(pSql, pEpSet); + } + + (*pSql->fp)(pSql->param, pSql, rpcMsg->code); + } + + if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it + tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); + taosRemoveRef(tscObjRef, handle); + } + + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); +} + +int doBuildAndSendMsg(SSqlObj *pSql) { + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; + + if (pCmd->command == TSDB_SQL_SELECT || + pCmd->command == TSDB_SQL_FETCH || + pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_INSERT || + pCmd->command == TSDB_SQL_CONNECT || + pCmd->command == TSDB_SQL_HB || + pCmd->command == TSDB_SQL_RETRIEVE_FUNC || + pCmd->command == TSDB_SQL_STABLEVGROUP) { + pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); + } + + if (pRes->code != TSDB_CODE_SUCCESS) { + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; + } + + int32_t code = tscSendMsgToServer(pSql); + + // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. + if (code != TSDB_CODE_SUCCESS) { + pRes->code = code; + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + +int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + + SSqlCmd *pCmd = &pSql->cmd; + uint32_t type = 0; + + if (pQueryInfo == NULL) { + pQueryInfo = tscGetQueryInfo(pCmd); + } + + STableMetaInfo *pTableMetaInfo = NULL; + + if (pQueryInfo != NULL) { + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo != NULL) { + tNameExtractFullName(&pTableMetaInfo->name, name); + } + + type = pQueryInfo->type; + + // while numOfTables equals to 0, it must be Heartbeat + assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0); + } + + tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type); + if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL + if (pTableMetaInfo == NULL) { + pSql->res.code = TSDB_CODE_TSC_APP_ERROR; + return pSql->res.code; + } + } else if (pCmd->command >= TSDB_SQL_LOCAL) { + return (*tscProcessMsgRsp[pCmd->command])(pSql); + } + + return doBuildAndSendMsg(pSql); +} + +int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + + pRetrieveMsg->free = htons(pQueryInfo->type); + pRetrieveMsg->qId = htobe64(pSql->res.qId); + + // todo valid the vgroupId at the client side + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + pRetrieveMsg->header.vgId = htonl(vgId); + tscDebug("0x%"PRIx64" build fetch msg from vgId:%d, vgIndex:%d, qId:0x%" PRIx64, pSql->self, vgId, vgIndex, pSql->res.qId); + } else { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); + tscDebug("0x%"PRIx64" build fetch msg from only one vgroup, vgId:%d, qId:0x%" PRIx64, pSql->self, pTableMeta->vgId, + pSql->res.qId); + } + + pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); + pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH; + + pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + // pSql->cmd.payloadLen is set during copying data into payload + pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); + + return TSDB_CODE_SUCCESS; +} + +/* + * for table query, simply return the size <= 1k + */ +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { + const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; + + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); + int32_t srcColFilterSize = 0; + int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); + + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); + + int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; + + int32_t tableSerialize = 0; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + if (pTableMetaInfo->pVgroupTables != NULL) { + size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + + int32_t totalTables = 0; + for (int32_t i = 0; i < numOfGroups; ++i) { + SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); + totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); + } + + tableSerialize = totalTables * sizeof(STableIdInfo); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + srcColFilterSize = pCond->len; + } + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + srcColListSize += pCond->len; + } + + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + + exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; +} + +static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, + int32_t *succeed) { + TSKEY dfltKey = htobe64(pQueryMsg->window.skey); + + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { + + int32_t vgId = -1; + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + int32_t index = pTableMetaInfo->vgroupIndex; + assert(index >= 0); + + SVgroupMsg* pVgroupInfo = NULL; + if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { + assert(index < pTableMetaInfo->vgroupList->numOfVgroups); + pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; + } else { + tscError("0x%"PRIx64" No vgroup info found", pSql->self); + + *succeed = 0; + return pMsg; + } + + vgId = pVgroupInfo->vgId; + tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); + tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); + } else { + vgId = pTableMeta->vgId; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + } + + if (pSql->epSet.numOfEps > 0){ + pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; + } + pQueryMsg->head.vgId = htonl(vgId); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pTableMeta->id.tid); + pTableIdInfo->uid = htobe64(pTableMeta->id.uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); + + pQueryMsg->numOfTables = htonl(1); // set the number of tables + pMsg += sizeof(STableIdInfo); + } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo + int32_t index = pTableMetaInfo->vgroupIndex; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(index >= 0 && index < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); + + // set the vgroup info + tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); + pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); + + int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); + pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables + + tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, + pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); + + // serialize each table id info + for(int32_t i = 0; i < numOfTables; ++i) { + STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pItem->tid); + pTableIdInfo->uid = htobe64(pItem->uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); + pMsg += sizeof(STableIdInfo); + } + } + + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + + tscDebug("0x%"PRIx64" vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql->self, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); + return pMsg; +} + +// TODO refactor +static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { + // append the filter information after the basic column information + for (int32_t f = 0; f < numOfFilters; ++f) { + SColumnFilterInfo *pColFilter = &pColFilters[f]; + + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); + pFilterMsg->filterstr = htons(pColFilter->filterstr); + + (*pMsg) += sizeof(SColumnFilterInfo); + + if (pColFilter->filterstr) { + pFilterMsg->len = htobe64(pColFilter->len); + memcpy(*pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); + (*pMsg) += (pColFilter->len + 1); // append the additional filter binary info + } else { + pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); + pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); + } + + pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); + pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); + + if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { + tscError("invalid filter info"); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, char** pMsg, int64_t id, bool validateColumn) { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + // the queried table has been removed and a new table with the same name has already been created already + // return error msg + if (pExpr->uid != pTableMeta->id.uid) { + tscError("0x%"PRIx64" table has already been destroyed", id); + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + + if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { + tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + assert(pExpr->resColId < 0); + SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); + + SColIndex* pIndex = &pSqlExpr->colInfo; + + pIndex->colId = htons(pExpr->colInfo.colId); + pIndex->colIndex = htons(pExpr->colInfo.colIndex); + pIndex->flag = htons(pExpr->colInfo.flag); + pSqlExpr->uid = htobe64(pExpr->uid); + pSqlExpr->colType = htons(pExpr->colType); + pSqlExpr->colBytes = htons(pExpr->colBytes); + pSqlExpr->resType = htons(pExpr->resType); + pSqlExpr->resBytes = htons(pExpr->resBytes); + pSqlExpr->interBytes = htonl(pExpr->interBytes); + pSqlExpr->functionId = htons(pExpr->functionId); + pSqlExpr->numOfParams = htons(pExpr->numOfParams); + pSqlExpr->resColId = htons(pExpr->resColId); + pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters); + + (*pMsg) += sizeof(SSqlExpr); + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log + pSqlExpr->param[j].nType = htonl(pExpr->param[j].nType); + pSqlExpr->param[j].nLen = htonl(pExpr->param[j].nLen); + + if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { + memcpy((*pMsg), pExpr->param[j].pz, pExpr->param[j].nLen); + (*pMsg) += pExpr->param[j].nLen; + } else { + pSqlExpr->param[j].i64 = htobe64(pExpr->param[j].i64); + } + } + + serializeColFilterInfo(pExpr->flist.filterInfo, pExpr->flist.numOfFilters, pMsg); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = NULL; + STableMeta *pTableMeta = NULL; + STableMetaInfo *pTableMetaInfo = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + int32_t size = tscEstimateQueryMsgSize(pSql); + assert(size > 0); + + if (TSDB_CODE_SUCCESS != tscAllocPayloadFast(pCmd, size)) { + tscError("%p failed to malloc for query msg", pSql); + return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this + } + + pQueryInfo = tscGetQueryInfo(pCmd); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + pTableMeta = pTableMetaInfo->pTableMeta; + + SQueryAttr query = {{0}}; + tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); + query.vgId = pTableMeta->vgId; + + SArray* tableScanOperator = createTableScanPlan(&query); + SArray* queryOperator = createExecOperatorPlan(&query); + + SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; + tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); + + int32_t numOfTags = query.numOfTags; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); + + if (taosArrayGetSize(tableScanOperator) == 0) { + pQueryMsg->tableScanOperator = htonl(-1); + } else { + int32_t* tablescanOp = taosArrayGet(tableScanOperator, 0); + pQueryMsg->tableScanOperator = htonl(*tablescanOp); + } + + pQueryMsg->window.skey = htobe64(query.window.skey); + pQueryMsg->window.ekey = htobe64(query.window.ekey); + + pQueryMsg->order = htons(query.order.order); + pQueryMsg->orderColId = htons(query.order.orderColId); + pQueryMsg->fillType = htons(query.fillType); + pQueryMsg->limit = htobe64(query.limit.limit); + pQueryMsg->offset = htobe64(query.limit.offset); + pQueryMsg->numOfCols = htons(query.numOfCols); + + pQueryMsg->interval.interval = htobe64(query.interval.interval); + pQueryMsg->interval.sliding = htobe64(query.interval.sliding); + pQueryMsg->interval.offset = htobe64(query.interval.offset); + pQueryMsg->interval.intervalUnit = query.interval.intervalUnit; + pQueryMsg->interval.slidingUnit = query.interval.slidingUnit; + pQueryMsg->interval.offsetUnit = query.interval.offsetUnit; + + pQueryMsg->stableQuery = query.stableQuery; + pQueryMsg->topBotQuery = query.topBotQuery; + pQueryMsg->groupbyColumn = query.groupbyColumn; + pQueryMsg->hasTagResults = query.hasTagResults; + pQueryMsg->timeWindowInterpo = query.timeWindowInterpo; + pQueryMsg->queryBlockDist = query.queryBlockDist; + pQueryMsg->stabledev = query.stabledev; + pQueryMsg->tsCompQuery = query.tsCompQuery; + pQueryMsg->simpleAgg = query.simpleAgg; + pQueryMsg->pointInterpQuery = query.pointInterpQuery; + pQueryMsg->needReverseScan = query.needReverseScan; + pQueryMsg->stateWindow = query.stateWindow; + pQueryMsg->numOfTags = htonl(numOfTags); + pQueryMsg->sqlstrLen = htonl(sqlLen); + pQueryMsg->sw.gap = htobe64(query.sw.gap); + pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); + + pQueryMsg->secondStageOutput = htonl(query.numOfExpr2); + pQueryMsg->numOfOutput = htons((int16_t)query.numOfOutput); // this is the stage one output column number + + pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); + pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); + pQueryMsg->queryType = htonl(pQueryInfo->type); + pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); + + // set column list ids + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + char *pMsg = (char *)(pQueryMsg->tableCols) + numOfCols * sizeof(SColumnInfo); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfo *pCol = &query.tableCols[i]; + + pQueryMsg->tableCols[i].colId = htons(pCol->colId); + pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); + pQueryMsg->tableCols[i].type = htons(pCol->type); + //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); + pQueryMsg->tableCols[i].flist.numOfFilters = 0; + pQueryMsg->tableCols[i].flist.filterInfo = 0; + // append the filter information after the basic column information + //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->colCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } + } else { + pQueryMsg->colCondLen = 0; + } + + for (int32_t i = 0; i < query.numOfOutput; ++i) { + code = serializeSqlExpr(&query.pExpr1[i].base, pTableMetaInfo, &pMsg, pSql->self, true); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + + for (int32_t i = 0; i < query.numOfExpr2; ++i) { + code = serializeSqlExpr(&query.pExpr2[i].base, pTableMetaInfo, &pMsg, pSql->self, false); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + + int32_t succeed = 1; + + // serialize the table info (sid, uid, tags) + pMsg = doSerializeTableInfo(pQueryMsg, pSql, pTableMetaInfo, pMsg, &succeed); + if (succeed == 0) { + code = TSDB_CODE_TSC_APP_ERROR; + goto _end; + } + + SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; + if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { + pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); + pQueryMsg->orderType = htons(pGroupbyExpr->orderType); + + for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { + SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); + + *((int16_t *)pMsg) = htons(pCol->colId); + pMsg += sizeof(pCol->colId); + + *((int16_t *)pMsg) += htons(pCol->colIndex); + pMsg += sizeof(pCol->colIndex); + + *((int16_t *)pMsg) += htons(pCol->flag); + pMsg += sizeof(pCol->flag); + + memcpy(pMsg, pCol->name, tListLen(pCol->name)); + pMsg += tListLen(pCol->name); + } + } + + if (query.fillType != TSDB_FILL_NONE) { + for (int32_t i = 0; i < query.numOfOutput; ++i) { + *((int64_t *)pMsg) = htobe64(query.fillVal[i]); + pMsg += sizeof(query.fillVal[0]); + } + } + + if (query.numOfTags > 0 && query.tagColList != NULL) { + for (int32_t i = 0; i < query.numOfTags; ++i) { + SColumnInfo* pTag = &query.tagColList[i]; + + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + pTagCol->colId = htons(pTag->colId); + pTagCol->bytes = htons(pTag->bytes); + pTagCol->type = htons(pTag->type); + pTagCol->flist.numOfFilters = 0; + + pMsg += sizeof(SColumnInfo); + } + } + + // serialize tag column query condition + if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { + STagCond* pTagCond = &pQueryInfo->tagCond; + + SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->tagCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } + } else { + pQueryMsg->tagCondLen = 0; + } + + if (pQueryInfo->bufLen > 0) { + memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); + pMsg += pQueryInfo->bufLen; + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + strncpy(pMsg, pCond->cond, pCond->len); + pMsg += pCond->len; + } + + // compressed ts block + pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); + + if (pQueryInfo->tsBuf != NULL) { + // note: here used the index instead of actual vnode id. + int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; + code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + pMsg += pQueryMsg->tsBuf.tsLen; + + pQueryMsg->tsBuf.tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); + pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); + pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); + } else { + pQueryMsg->tsBuf.tsLen = 0; + pQueryMsg->tsBuf.tsNumOfBlocks = 0; + } + + int32_t numOfOperator = (int32_t) taosArrayGetSize(queryOperator); + pQueryMsg->numOfOperator = htonl(numOfOperator); + for(int32_t i = 0; i < numOfOperator; ++i) { + int32_t *operator = taosArrayGet(queryOperator, i); + *(int32_t*)pMsg = htonl(*operator); + + pMsg += sizeof(int32_t); + } + + // support only one udf + if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); + for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); + *(int8_t*) pMsg = pUdfInfo->resType; + pMsg += sizeof(pUdfInfo->resType); + + *(int16_t*) pMsg = htons(pUdfInfo->resBytes); + pMsg += sizeof(pUdfInfo->resBytes); + + STR_TO_VARSTR(pMsg, pUdfInfo->name); + + pMsg += varDataTLen(pMsg); + + *(int32_t*) pMsg = htonl(pUdfInfo->funcType); + pMsg += sizeof(pUdfInfo->funcType); + + *(int32_t*) pMsg = htonl(pUdfInfo->bufSize); + pMsg += sizeof(pUdfInfo->bufSize); + + pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen); + memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen); + + pMsg += pUdfInfo->contLen; + } + } else { + pQueryMsg->udfContentOffset = 0; + pQueryMsg->udfContentLen = 0; + } + + memcpy(pMsg, pSql->sqlstr, sqlLen); + pMsg += sqlLen; + + int32_t msgLen = (int32_t)(pMsg - pCmd->payload); + + tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); + pCmd->payloadLen = msgLen; + pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; + + pQueryMsg->head.contLen = htonl(msgLen); + assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); + + _end: + freeQueryAttr(&query); + taosArrayDestroy(tableScanOperator); + taosArrayDestroy(queryOperator); + return code; +} + +int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateDbMsg); + + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; + + SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; + +// assert(pCmd->numOfClause == 1); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); + assert(code == TSDB_CODE_SUCCESS); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_FUNCTION; + + pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateDnodeMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; + + SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); + strncpy(pCreate->ep, t0->z, t0->n); + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateAcctMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; + + SStrToken *pName = &pInfo->pMiscInfo->user.user; + SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; + + strncpy(pAlterMsg->user, pName->z, pName->n); + strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); + + SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; + + pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); + pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); + pAlterMsg->cfg.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); + pAlterMsg->cfg.maxStreams = htonl(pAcctOpt->maxStreams); + pAlterMsg->cfg.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); + pAlterMsg->cfg.maxStorage = htobe64(pAcctOpt->maxStorage); + pAlterMsg->cfg.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); + pAlterMsg->cfg.maxConnections = htonl(pAcctOpt->maxConnections); + + if (pAcctOpt->stat.n == 0) { + pAlterMsg->cfg.accessState = -1; + } else { + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + pAlterMsg->cfg.accessState = TSDB_VN_READ_ACCCESS; + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + pAlterMsg->cfg.accessState = TSDB_VN_WRITE_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + pAlterMsg->cfg.accessState = TSDB_VN_ALL_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + pAlterMsg->cfg.accessState = 0; + } + } + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateUserMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; + + SUserInfo *pUser = &pInfo->pMiscInfo->user; + strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); + pAlterMsg->flag = (int8_t)pUser->type; + + if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + pAlterMsg->privilege = (char)pCmd->count; + } else if (pUser->type == TSDB_ALTER_USER_PASSWD) { + strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); + } else { // create user password info + strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); + } + + if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + pCmd->msgType = TSDB_MSG_TYPE_ALTER_USER; + } else { + pCmd->msgType = TSDB_MSG_TYPE_CREATE_USER; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCfgDnodeMsg); + pCmd->msgType = TSDB_MSG_TYPE_CONFIG_DNODE; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SDropDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); + assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); + + pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + + pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_DROP_DB : TSDB_MSG_TYPE_DROP_TP; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->msgType = TSDB_MSG_TYPE_DROP_FUNCTION; + + pCmd->payloadLen = sizeof(SDropFuncMsg); + + return TSDB_CODE_SUCCESS; +} + + +int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCMDropTableMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); + + pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; + pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + pCmd->msgType = TSDB_MSG_TYPE_DROP_TABLE; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char dnodeEp[TSDB_EP_LEN] = {0}; + tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); + + pCmd->payloadLen = sizeof(SDropDnodeMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; + tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); + pCmd->msgType = TSDB_MSG_TYPE_DROP_DNODE; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char user[TSDB_USER_LEN] = {0}; + tstrncpy(user, pCmd->payload, TSDB_USER_LEN); + + pCmd->payloadLen = sizeof(SDropUserMsg); + pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_DROP_USER:TSDB_MSG_TYPE_DROP_ACCT; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropUserMsg *pDropMsg = (SDropUserMsg *)pCmd->payload; + tstrncpy(pDropMsg->user, user, tListLen(user)); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SUseDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_USE_DB; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SSyncDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_SYNC_DB; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_SHOW; + pCmd->payloadLen = sizeof(SShowMsg) + 100; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; + SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { + pShowMsg->type = pShowInfo->showType; + pShowMsg->payloadLen = 0; + pCmd->payloadLen = sizeof(SShowMsg); + + return TSDB_CODE_SUCCESS; + } + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + char *p = cloneCurrentDBName(pSql); + tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db)); + tfree(p); + } else { + tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); + } + + pShowMsg->type = pShowInfo->showType; + + if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { + SStrToken *pPattern = &pShowInfo->pattern; + if (pPattern->type > 0) { // only show tables support wildcard query + strncpy(pShowMsg->payload, pPattern->z, pPattern->n); + pShowMsg->payloadLen = htons(pPattern->n); + } + } else { + SStrToken *pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); + + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); + } + + pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SKillQueryMsg); + + switch (pCmd->command) { + case TSDB_SQL_KILL_QUERY: + pCmd->msgType = TSDB_MSG_TYPE_KILL_QUERY; + break; + case TSDB_SQL_KILL_CONNECTION: + pCmd->msgType = TSDB_MSG_TYPE_KILL_CONN; + break; + case TSDB_SQL_KILL_STREAM: + pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; + break; + } + return TSDB_CODE_SUCCESS; +} + +int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &(pSql->cmd); + int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); + + SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; + if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { + int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); + size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); + } else { + size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); + } + + if (pCreateTableInfo->pSelect != NULL) { + size += (pCreateTableInfo->pSelect->sqlstr.n + 1); + } + + return size + TSDB_EXTRA_PAYLOAD_SIZE; +} + +int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + int msgLen = 0; + int size = 0; + SSchema *pSchema; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + // Reallocate the payload size + size = tscEstimateCreateTableMsgLength(pSql, pInfo); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; + + SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); + char* pMsg = NULL; + + int8_t type = pInfo->pCreateTableInfo->type; + if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value + SArray* list = pInfo->pCreateTableInfo->childTableInfo; + + int32_t numOfTables = (int32_t) taosArrayGetSize(list); + pCreateTableMsg->numOfTables = htonl(numOfTables); + + pMsg = (char*) pCreateMsg; + for(int32_t i = 0; i < numOfTables; ++i) { + SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; + + pCreate->numOfColumns = htons(pCmd->numOfCols); + pCreate->numOfTags = htons(pCmd->count); + pMsg += sizeof(SCreateTableMsg); + + SCreatedTableInfo* p = taosArrayGet(list, i); + strcpy(pCreate->tableName, p->fullname); + pCreate->igExists = (p->igExist)? 1 : 0; + + // use dbinfo from table id without modifying current db info + pMsg = serializeTagData(&p->tagdata, pMsg); + + int32_t len = (int32_t)(pMsg - (char*) pCreate); + pCreate->len = htonl(len); + } + } else { // create (super) table + pCreateTableMsg->numOfTables = htonl(1); // only one table will be created + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); + assert(code == 0); + + SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; + + pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; + pCreateMsg->numOfColumns = htons(pCmd->numOfCols); + pCreateMsg->numOfTags = htons(pCmd->count); + + pCreateMsg->sqlLen = 0; + pMsg = (char *)pCreateMsg->schema; + + pSchema = (SSchema *)pCreateMsg->schema; + + for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + + pSchema->type = pField->type; + strcpy(pSchema->name, pField->name); + pSchema->bytes = htons(pField->bytes); + + pSchema++; + } + + pMsg = (char *)pSchema; + if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql + SSqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; + + strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); + pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); + pMsg += pQuerySql->sqlstr.n + 1; + } + } + + tscFieldInfoClear(&pQueryInfo->fieldsInfo); + + msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); + pCreateTableMsg->contLen = htonl(msgLen); + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + + assert(msgLen + minMsgSize() <= size); + return TSDB_CODE_SUCCESS; +} + +int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; +} + +int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + char *pMsg; + int msgLen = 0; + + SSqlCmd *pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; + int size = tscEstimateAlterTableMsgLength(pCmd); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; + + tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); + pAlterTableMsg->type = htons(pAlterInfo->type); + + pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); + SSchema *pSchema = pAlterTableMsg->schema; + for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + + pSchema->type = pField->type; + strcpy(pSchema->name, pField->name); + pSchema->bytes = htons(pField->bytes); + pSchema++; + } + + pMsg = (char *)pSchema; + pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); + if (pAlterInfo->tagData.dataLen > 0) { + memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); + } + pMsg += pAlterInfo->tagData.dataLen; + + msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); + + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_ALTER_TABLE; + + assert(msgLen + minMsgSize() <= size); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { + SSqlCmd* pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL; + + SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; + pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + assert(vgroupInfo.vgId > 0); + + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + return TSDB_CODE_SUCCESS; +} + +int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SAlterDbMsg); + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; + + SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; + pAlterDbMsg->dbType = -1; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); + + return TSDB_CODE_SUCCESS; +} +int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + if (pInfo->list == NULL || taosArrayGetSize(pInfo->list) <= 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + STscObj *pObj = pSql->pTscObj; + SSqlCmd *pCmd = &pSql->cmd; + SArray *pList = pInfo->list; + int32_t size = (int32_t)taosArrayGetSize(pList); + + int32_t *result = malloc(sizeof(int32_t) * size); + if (result == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < size; i++) { + tSqlExprItem* pSub = taosArrayGet(pList, i); + tVariant* pVar = &pSub->pNode->value; + if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { + result[i] = (int32_t)(pVar->i64); + } else { + free(result); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + int count = removeDupVgid(result, size); + pCmd->payloadLen = sizeof(SCompactMsg) + count * sizeof(int32_t); + pCmd->msgType = TSDB_MSG_TYPE_COMPACT_VNODE; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + free(result); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SCompactMsg *pCompactMsg = (SCompactMsg *)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + pthread_mutex_lock(&pObj->mutex); + tstrncpy(pCompactMsg->db, pObj->db, sizeof(pCompactMsg->db)); + pthread_mutex_unlock(&pObj->mutex); + } else { + tNameGetFullDbName(&pTableMetaInfo->name, pCompactMsg->db); + } + + pCompactMsg->numOfVgroup = htons(count); + for (int32_t i = 0; i < count; i++) { + pCompactMsg->vgid[i] = htons(result[i]); + } + free(result); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pCmd->payloadLen = sizeof(SRetrieveTableMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; + pRetrieveMsg->qId = htobe64(pSql->res.qId); + pRetrieveMsg->free = htons(pQueryInfo->type); + + return TSDB_CODE_SUCCESS; +} + +/* + * this function can only be called once. + * by using pRes->rspType to denote its status + * + * if pRes->rspType is 1, no more result + */ +static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pRes->code = TSDB_CODE_SUCCESS; + if (pRes->rspType == 0) { + pRes->numOfRows = numOfRes; + pRes->row = 0; + pRes->rspType = 1; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + tscSetResRawPtr(pRes, pQueryInfo); + } else { + tscResetForNextRetrieve(pRes); + } + + uint8_t code = pSql->res.code; + if (pSql->fp) { + if (code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); + } else { + tscAsyncResultOnError(pSql); + } + } + + return code; +} + +int tscProcessDescribeTableRsp(SSqlObj *pSql) { + SSqlCmd * pCmd = &pSql->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; + return tscLocalResultCommonBuilder(pSql, numOfRes); +} + +int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { + int32_t numOfRes = 1; + pSql->res.completed = true; + return tscLocalResultCommonBuilder(pSql, numOfRes); +} + +int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd* pCmd = &pSql->cmd; + + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscAsyncResultOnError(pSql); + return code; + } + + if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly. + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + return code; + } + + // global aggregation may be the upstream for parent query + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo->pQInfo == NULL) { + STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; + tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + + STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; + + SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(group, &tableKeyInfo); + taosArrayPush(tableGroupInfo.pGroupList, &group); + + tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute query processing", pSql->self, pSql->self); + pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE, pSql->self); + } + + uint64_t localQueryId = pSql->self; + qTableQuery(pQueryInfo->pQInfo, &localQueryId); + convertQueryResult(pRes, pQueryInfo, pSql->self, true); + + code = pRes->code; + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + } else { + tscAsyncResultOnError(pSql); + } + + return code; +} + +int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } + +int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + STscObj *pObj = pSql->pTscObj; + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_CONNECT; + pCmd->payloadLen = sizeof(SConnectMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; + + // TODO refactor full_name + char *db; // ugly code to move the space + + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); + tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); + + pConnect->pid = htonl(taosGetPId()); + taosGetCurrentAPPName(pConnect->appName, NULL); + + return TSDB_CODE_SUCCESS; +} + +/** + * multi table meta req pkg format: + * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... + * 4B + **/ +int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; + assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); + + tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, + pCmd->payloadLen); + + return pCmd->payloadLen; +} + +int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char* pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; + pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); + pMsg += sizeof(SSTableVgroupMsg); + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); + assert(code == TSDB_CODE_SUCCESS); + + pMsg += TSDB_TABLE_FNAME_LEN; + } + + pCmd->msgType = TSDB_MSG_TYPE_STABLE_VGROUP; + pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char *pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); + + SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; + pRetrieveFuncMsg->num = htonl(numOfFuncs); + + pMsg += sizeof(SRetrieveFuncMsg); + for(int32_t i = 0; i < numOfFuncs; ++i) { + SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i); + STR_TO_NET_VARSTR(pMsg, pUdf->name); + pMsg += varDataNetTLen(pMsg); + } + + pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE_FUNC; + pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + STscObj *pObj = pSql->pTscObj; + + pthread_mutex_lock(&pObj->mutex); + + int32_t numOfQueries = 2; + SSqlObj *tpSql = pObj->sqlList; + while (tpSql) { + tpSql = tpSql->next; + numOfQueries++; + } + + int32_t numOfStreams = 2; + SSqlStream *pStream = pObj->streamList; + while (pStream) { + pStream = pStream->next; + numOfStreams++; + } + + int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + pthread_mutex_unlock(&pObj->mutex); + tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + // TODO the expired hb and client can not be identified by server till now. + SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; + tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); + + pHeartbeat->numOfQueries = numOfQueries; + pHeartbeat->numOfStreams = numOfStreams; + + pHeartbeat->pid = htonl(taosGetPId()); + taosGetCurrentAPPName(pHeartbeat->appName, NULL); + + int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); + + pthread_mutex_unlock(&pObj->mutex); + + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); + + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); + + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && + (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, + pMetaMsg->tid, pMetaMsg->tableFname); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) { + tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { + tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { + pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); + } + + SSchema* pSchema = pMetaMsg->schema; + + int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; + for (int i = 0; i < numOfTotalCols; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema->colId = htons(pSchema->colId); + + if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + assert(i == 0); + } + + pSchema++; + } + + return TSDB_CODE_SUCCESS; +} + +// update the vgroupInfo if needed +static void doUpdateVgroupInfo(int32_t vgId, SVgroupMsg *pVgroupMsg) { + assert(vgId > 0); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); + + // vgroup info exists, compare with it + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { + vgroupInfo = createNewVgroupInfo(pVgroupMsg); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add/update new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } +} + +static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + // add or update the corresponding super table meta data info + int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + // The super tableMeta already exists, create it according to tableMeta and add it to hash map + if (updateSTable) { + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaMap, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); + + tfree(pSupTableMeta); + } + + CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); + taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); + tfree(cMeta); + } else { + uint32_t s = tscGetTableMetaSize(pTableMeta); + taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); + } +} + +int tscProcessTableMetaRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + assert(pTableMetaInfo->pTableMeta == NULL); + + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (pTableMeta == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); + tfree(pTableMeta); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); + + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); + if (pTableMeta->tableType != TSDB_SUPER_TABLE) { + doUpdateVgroupInfo(pTableMeta->vgId, &pMetaMsg->vgroup); + } + + tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, + pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, + pTableMeta->tableInfo.numOfTags); + + free(pTableMeta); + return TSDB_CODE_SUCCESS; +} + +static SArray* createVgroupIdListFromMsg(char* pMsg, SHashObj* pSet, char* name, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); + + SArray* vgroupIdList = taosArrayInit(pVgroupMsg->numOfVgroups, sizeof(int32_t)); + + if (pVgroupMsg->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup id list, no corresponding tables for stable:%s", id, name); + } else { + // just init, no need to lock + for (int32_t j = 0; j < pVgroupMsg->numOfVgroups; ++j) { + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } + + taosArrayPush(vgroupIdList, &vmsg->vgId); + + if (taosHashGet(pSet, &vmsg->vgId, sizeof(vmsg->vgId)) == NULL) { + taosHashPut(pSet, &vmsg->vgId, sizeof(vmsg->vgId), "", 0); + doUpdateVgroupInfo(vmsg->vgId, vmsg); + } + } + } + + return vgroupIdList; +} + +static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); + + size_t vgroupsz = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); + SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); + assert(pVgroupInfo != NULL); + + pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; + if (pVgroupInfo->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); + } else { + for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { + // just init, no need to lock + SVgroupMsg *pVgroup = &pVgroupInfo->vgroups[j]; + + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } + + pVgroup->numOfEps = vmsg->numOfEps; + pVgroup->vgId = vmsg->vgId; + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + pVgroup->epAddr[k].port = vmsg->epAddr[k].port; + tstrncpy(pVgroup->epAddr[k].fqdn, vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); +// pVgroup->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); + } + + doUpdateVgroupInfo(pVgroup->vgId, vmsg); + } + } + + return pVgroupInfo; +} + +int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + pFuncMsg->num = htonl(pFuncMsg->num); + assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); + + char* pMsg = pFuncMsg->content; + for(int32_t i = 0; i < pFuncMsg->num; ++i) { + SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; + + for(int32_t j = 0; j < pFuncMsg->num; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + if (strcmp(pUdfInfo->name, pFunc->name) != 0) { + continue; + } + + if (pUdfInfo->content) { + continue; + } + + pUdfInfo->resBytes = htons(pFunc->resBytes); + pUdfInfo->resType = pFunc->resType; + pUdfInfo->funcType = htonl(pFunc->funcType); + pUdfInfo->contLen = htonl(pFunc->len); + pUdfInfo->bufSize = htonl(pFunc->bufSize); + + pUdfInfo->content = malloc(pUdfInfo->contLen); + memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); + + pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; + } + } + + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + taosArrayDestroy(parQueryInfo->pUdfInfo); + + parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj. + pQueryInfo->pUdfInfo = NULL; + return TSDB_CODE_SUCCESS; +} + +int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { + char *rsp = pSql->res.pRsp; + + SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; + pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); + pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); + pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf); + + rsp += sizeof(SMultiTableMeta); + + SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(pParentSql == NULL) { + return pSql->res.code; + } + + SSqlCmd *pParentCmd = &pParentSql->cmd; + SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + + char* buf = NULL; + char* pMsg = pMultiMeta->meta; + + // decompresss the message payload + if (pMultiMeta->compressed) { + buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, + buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); + assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + + pMsg = buf; + } + + if (pParentCmd->pTableMetaMap == NULL) { + pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + } + + for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return code; + } + + bool freeMeta = false; + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); + tfree(pTableMeta); + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { + STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; + size_t keyLen = strnlen(pMetaMsg->tableFname, TSDB_TABLE_FNAME_LEN); + void* t = taosHashGet(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen); + assert(t == NULL); + + taosHashPut(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen, &p, sizeof(STableMetaVgroupInfo)); + } else { + freeMeta = true; + } + + // for each super table, only update meta information once + bool updateStableMeta = false; + if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { + updateStableMeta = true; + taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); + } + + // create the tableMeta and add it into the TableMeta map + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); + + // for each vgroup, only update the information once. + int64_t vgId = pMetaMsg->vgroup.vgId; + if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { + doUpdateVgroupInfo((int32_t) vgId, &pMetaMsg->vgroup); + taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); + } + + pMsg += pMetaMsg->contLen; + if (freeMeta) { + tfree(pTableMeta); + } + } + + for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { + char fname[TSDB_TABLE_FNAME_LEN] = {0}; + tstrncpy(fname, pMsg, TSDB_TABLE_FNAME_LEN); + size_t len = strnlen(fname, TSDB_TABLE_FNAME_LEN); + + pMsg += TSDB_TABLE_FNAME_LEN; + + STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, fname, len); + assert(p != NULL); + + int32_t size = 0; + if (p->vgroupIdList!= NULL) { + taosArrayDestroy(p->vgroupIdList); + } + + p->vgroupIdList = createVgroupIdListFromMsg(pMsg, pSet, fname, &size, pSql->self); + + int32_t numOfVgId = (int32_t) taosArrayGetSize(p->vgroupIdList); + int32_t s = sizeof(tFilePage) + numOfVgId * sizeof(int32_t); + + tFilePage* idList = calloc(1, s); + idList->num = numOfVgId; + memcpy(idList->data, TARRAY_GET_START(p->vgroupIdList), numOfVgId * sizeof(int32_t)); + + void* idListInst = taosCachePut(tscVgroupListBuf, fname, len, idList, s, 5000); + taosCacheRelease(tscVgroupListBuf, (void*) &idListInst, false); + + tfree(idList); + pMsg += size; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd); + if (pMultiMeta->numOfUdf > 0) { + assert(pQueryInfo->pUdfInfo != NULL); + } + + for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) { + SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; + + for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + if (strcmp(pUdfInfo->name, pFunc->name) != 0) { + continue; + } + + if (pUdfInfo->content) { + continue; + } + + pUdfInfo->resBytes = htons(pFunc->resBytes); + pUdfInfo->resType = pFunc->resType; + pUdfInfo->funcType = htonl(pFunc->funcType); + pUdfInfo->contLen = htonl(pFunc->len); + pUdfInfo->bufSize = htonl(pFunc->bufSize); + + pUdfInfo->content = malloc(pUdfInfo->contLen); + memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); + + pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; + } + } + + pSql->res.code = TSDB_CODE_SUCCESS; + pSql->res.numOfTotal = pMultiMeta->numOfTables; + tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); + + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return TSDB_CODE_SUCCESS; +} + +int tscProcessSTableVgroupRsp(SSqlObj *pSql) { + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + + SSqlRes* pRes = &pSql->res; + + // NOTE: the order of several table must be preserved. + SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; + pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); + char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); + + SSqlCmd* pCmd = &parent->cmd; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + char fName[TSDB_TABLE_FNAME_LEN] = {0}; + for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { + char* name = pMsg; + pMsg += TSDB_TABLE_FNAME_LEN; + + STableMetaInfo *pInfo = NULL; + for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { + STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); + memset(fName, 0, tListLen(fName)); + + tNameExtractFullName(&pInfo1->name, fName); + if (strcmp(name, fName) != 0) { + continue; + } + + pInfo = pInfo1; + break; + } + + if (!pInfo){ + continue; + } + int32_t size = 0; + pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); + pMsg += size; + } + + taosReleaseRef(tscObjRef, parent->self); + return pSql->res.code; +} + +int tscProcessShowRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg; + SShowRsp * pShow; + SSchema * pSchema; + + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + pShow = (SShowRsp *)pRes->pRsp; + pShow->qhandle = htobe64(pShow->qhandle); + pRes->qId = pShow->qhandle; + + tscResetForNextRetrieve(pRes); + pMetaMsg = &(pShow->tableMeta); + + pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); + + pSchema = pMetaMsg->schema; + pMetaMsg->tid = ntohs(pMetaMsg->tid); + for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema++; + } + + tfree(pTableMetaInfo->pTableMeta); + pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + + SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + SColumnIndex index = {0}; + pSchema = pMetaMsg->schema; + + uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; + for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { + index.columnIndex = i; + tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); + + TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); + SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); + + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, + pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); + } + + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + tscFieldInfoUpdateOffset(pQueryInfo); + return 0; +} + +static void createHbObj(STscObj* pObj) { + if (pObj->hbrid != 0) { + return; + } + + SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); + if (NULL == pSql) return; + + pSql->fp = tscProcessHeartBeatRsp; + + SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); + if (pQueryInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); + return; + } + + pQueryInfo->command = TSDB_SQL_HB; + + pSql->cmd.command = pQueryInfo->command; + if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); + return; + } + + pSql->param = pObj; + pSql->pTscObj = pObj; + pSql->signature = pSql; + + registerSqlObj(pSql); + tscDebug("0x%"PRIx64" HB is allocated, pObj:%p", pSql->self, pObj); + + pObj->hbrid = pSql->self; +} + +int tscProcessUseDbRsp(SSqlObj *pSql) { + STscObj * pObj = pSql->pTscObj; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + + pthread_mutex_lock(&pObj->mutex); + int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + pthread_mutex_unlock(&pObj->mutex); + + return ret; +} + +//todo only invalid the buffered data that belongs to dropped databases +int tscProcessDropDbRsp(SSqlObj *pSql) { + //TODO LOCK DB WHEN MODIFY IT + //pSql->pTscObj->db[0] = 0; + + taosHashClear(tscTableMetaMap); + taosHashClear(tscVgroupMap); + taosCacheEmpty(tscVgroupListBuf); + return 0; +} + +int tscProcessDropTableRsp(SSqlObj *pSql) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); + tfree(pTableMetaInfo->pTableMeta); + return 0; +} + +int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + tscDebug("0x%"PRIx64" remove tableMeta in hashMap after alter-table: %s", pSql->self, name); + + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tfree(pTableMetaInfo->pTableMeta); + + if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta + if (pSql->res.pRsp == NULL) { + tscDebug("0x%"PRIx64" unexpected resp from mnode, super table: %s failed to update super table meta ", pSql->self, name); + return 0; + } + return tscProcessTableMetaRsp(pSql); + } + + return 0; +} + +int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { + UNUSED(pSql); + return 0; +} +int tscProcessCompactRsp(SSqlObj *pSql) { + UNUSED(pSql); + return TSDB_CODE_SUCCESS; +} + +int tscProcessShowCreateRsp(SSqlObj *pSql) { + return tscLocalResultCommonBuilder(pSql, 1); +} + +int tscProcessQueryRsp(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + + SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp; + pQueryAttr->qId = htobe64(pQueryAttr->qId); + + pRes->qId = pQueryAttr->qId; + pRes->data = NULL; + + tscResetForNextRetrieve(pRes); + tscDebug("0x%"PRIx64" query rsp received, qId:0x%"PRIx64, pSql->self, pRes->qId); + return 0; +} + +static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { + int32_t decompLen = 0; + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + int32_t *compSizes; + char *pData = *data; + compSizes = (int32_t *)(pData + compLen); + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); + int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); + char *outputBuf = tcalloc(pRes->numOfRows, (pField->bytes + offset)); + + char *p = outputBuf; + int32_t bufOffset; + for (int32_t i = 0; i < numOfCols; ++i) { + SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); + bufOffset = pInfo->field.bytes * pRes->numOfRows; + + int32_t flen = (*(tDataTypes[pInfo->field.type].decompFunc))(pData, htonl(compSizes[i]), pRes->numOfRows, p, bufOffset, + compressed, NULL, 0); + + p += flen; + decompLen +=flen; + pData += htonl(compSizes[i]); + } + + /* Resize rsp as decompressed data will occupy more space */ + pRes->rspLen = pRes->rspLen - (compLen + numOfCols * sizeof(int32_t)) + decompLen; + char *new_rsp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (new_rsp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return; + } else { + pRes->pRsp = new_rsp; + *data = ((SRetrieveTableRsp *)pRes->pRsp)->data; + pData = *data + compLen + numOfCols * sizeof(int32_t); + } + + tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", + pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); + + int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; + memmove(*data + decompLen, pData, tailLen); + memmove(*data, outputBuf, decompLen); + + tfree(outputBuf); +} + +int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); + + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; + if (pRetrieve == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; + } + + pRes->numOfRows = htonl(pRetrieve->numOfRows); + pRes->precision = htons(pRetrieve->precision); + pRes->offset = htobe64(pRetrieve->offset); + pRes->useconds = htobe64(pRetrieve->useconds); + pRes->completed = (pRetrieve->completed == 1); + pRes->data = pRetrieve->data; + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + //Decompress col data if compressed from server + if (pRetrieve->compressed) { + int32_t compLen = htonl(pRetrieve->compLen); + decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); + } + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if ((pCmd->command == TSDB_SQL_RETRIEVE) || + ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || + (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { + tscSetResRawPtr(pRes, pQueryInfo); + } + + if (pSql->pSubscription != NULL) { + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); + int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); + + char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; + + int32_t numOfTables = htonl(*(int32_t*)p); + p += sizeof(int32_t); + for (int i = 0; i < numOfTables; i++) { + int64_t uid = htobe64(*(int64_t*)p); + p += sizeof(int64_t); + p += sizeof(int32_t); // skip tid + TSKEY key = htobe64(*(TSKEY*)p); + p += sizeof(TSKEY); + tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); + } + } + + pRes->row = 0; + tscDebug("0x%"PRIx64" numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:0x%"PRIx64, pSql->self, pRes->numOfRows, pRes->offset, + pRes->completed, pRes->qId); + + return 0; +} + +void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); + +static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_META; + + tscAddQueryInfo(&pNew->cmd); + + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); + assert(pNewQueryInfo->numOfTables == 1); + + tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); + + registerSqlObj(pNew); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, + pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); + pSql->metaRid = pNew->self; + + { + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; + int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + pInfoMsg->createFlag = htons(autocreate? 1 : 0); + char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); + + // tag data exists + if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) { + pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg); + } + + pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); + pNew->cmd.msgType = TSDB_MSG_TYPE_TABLE_META; + } + + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_MULTI_META; + + int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); + int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); + int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0; + + int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; + pInfo->metaClone = metaClone? 1:0; + pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); + pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); + pInfo->numOfUdfs = htonl(numOfUdf); + + char* start = pInfo->tableNames; + int32_t len = 0; + for(int32_t i = 0; i < numOfTable; ++i) { + char* name = taosArrayGetP(pNameList, i); + if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfVgroupList; ++i) { + char* name = taosArrayGetP(pVgroupNameList, i); + if (i < numOfVgroupList - 1 || numOfUdf > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfUdf; ++i) { + SUdfInfo * u = taosArrayGet(pUdfList, i); + if (i < numOfUdf - 1) { + len = sprintf(start, "%s,", u->name); + } else { + len = sprintf(start, "%s", u->name); + } + + start += len; + } + + pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); + pNew->cmd.msgType = TSDB_MSG_TYPE_TABLES_META; + + registerSqlObj(pNew); + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, + pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen); + + pNew->fp = fp; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" metaRid from 0x%" PRIx64 " to 0x%" PRIx64 , pSql->self, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { + assert(tIsValidName(&pTableMetaInfo->name)); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + size_t len = strlen(name); + // just make runtime happy + if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { + memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); + } + + if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity)) { + tfree(pTableMetaInfo->pTableMeta); + } + + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); + + if (pMeta && pMeta->id.uid > 0) { + // in case of child table, here only get the + if (pMeta->tableType == TSDB_CHILD_TABLE) { + int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); + pSql->pBuf = (void *)(pSTMeta); + if (code != TSDB_CODE_SUCCESS) { + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); + } + } + + tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); + return TSDB_CODE_SUCCESS; + } + + if (onlyLocal) { + return TSDB_CODE_TSC_NO_META_CACHED; + } + + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); +} + +int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, false, false); +} + +int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists, onlyLocal); +} + +int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC; + + if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) { + tscError("%p malloc failed for new queryinfo", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); + + pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); + for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { + SUdfInfo info = {0}; + SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i); + info = *p1; + info.name = strdup(p1->name); + taosArrayPush(pNewQueryInfo->pUdfInfo, &info); + } + + pNew->cmd.active = pNewQueryInfo; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { + tscError("%p malloc failed for payload to get table meta", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew); + registerSqlObj(pNew); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + + tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +static void freeElem(void* p) { + tfree(*(char**)p); +} + +/** + * retrieve table meta from mnode, and then update the local table meta hashmap. + * @param pSql sql object + * @param tableIndex table index + * @return status code + */ +int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { + SSqlCmd* pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); + if (code != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pTableMeta) { + tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); + } + + + // remove stored tableMeta info in hash table + tscResetSqlCmd(pCmd, true, pSql->self); + + SArray* pNameList = taosArrayInit(1, POINTER_BYTES); + SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); + + char* n = strdup(name); + taosArrayPush(pNameList, &n); + code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); + taosArrayDestroyEx(pNameList, freeElem); + taosArrayDestroyEx(vgroupList, freeElem); + + return code; +} + +static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (pTableMetaInfo->vgroupList == NULL) { + return false; + } + } + + // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore + return true; +} + +int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (allVgroupInfoRetrieved(pQueryInfo)) { + return TSDB_CODE_SUCCESS; + } + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + + pNew->cmd.command = TSDB_SQL_STABLEVGROUP; + + // TODO TEST IT + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); + if (pNewQueryInfo == NULL) { + tscFreeSqlObj(pNew); + return code; + } + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); + STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); + tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); + } + + if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pNew); + return code; + } + + pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + registerSqlObj(pNew); + + tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self); + + pSql->svgroupRid = pNew->self; + tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; + } + + return code; +} + +#endif + +int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + STscObj *pTscObj = pRequest->pTscObj; + + SConnectRsp *pConnect = (SConnectRsp *)pMsg; + pConnect->acctId = htonl(pConnect->acctId); + pConnect->connId = htonl(pConnect->connId); + pConnect->clusterId = htonl(pConnect->clusterId); + + // TODO refactor + pthread_mutex_lock(&pTscObj->mutex); + char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; + int32_t len = sprintf(temp, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db); + + assert(len <= sizeof(pTscObj->db)); + tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db)); + pthread_mutex_unlock(&pTscObj->mutex); + + assert(pConnect->epSet.numOfEps > 0); + if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet); + } + + for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id); + } + + pTscObj->connId = pConnect->connId; + + // update the appInstInfo + pTscObj->pAppInfo->clusterId = pConnect->clusterId; + atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + + tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); + // createHbObj(pTscObj); + + // launch a timer to send heartbeat to maintain the connection and send status to mnode + // taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer); + return 0; +} + +void initMsgHandleFp() { +#if 0 + tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; + tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; + tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; + + tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; + tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; + + tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; + tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; + + tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; + tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; + tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; + tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; + tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; + tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; + tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; + tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; + tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; + + + tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; + + tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; + tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; + + tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; + + tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; + tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; + + tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; + tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; + + tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; + + tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; + tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; + tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; +#endif + +// buildRequestMsgFp[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; +} \ No newline at end of file diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c new file mode 100644 index 0000000000000000000000000000000000000000..ba801358502b0a48a102afc338c412f7a00aa657 --- /dev/null +++ b/source/client/src/clientmain.c @@ -0,0 +1,85 @@ +#include "clientInt.h" +#include "trpc.h" +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "tscLog.h" +#include "tsched.h" +#include "ttime.h" +#include "ttimezone.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +static int32_t sentinel = TSC_VAR_NOT_RELEASE; +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; + +extern int32_t tscInitRes; + +int taos_options(TSDB_OPTION option, const void *arg, ...) { + static int32_t lock = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscInfo("haven't acquire lock after spin %d times.", i); + sched_yield(); + } + } + + int ret = taos_options_imp(option, (const char*)arg); + + atomic_store_32(&lock, 0); + return ret; +} + +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); + + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; + } + + int32_t id = tscReqRef; + tscReqRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscConnRef; + tscConnRef = -1; + taosCloseRef(id); + + rpcCleanup(); + taosCloseLog(); +} + +void taos_close(TAOS* taos) { + if (taos == NULL) { + return; + } + + STscObj *pTscObj = (STscObj *)taos; + tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + + taosRemoveRef(tscConnRef, pTscObj->id); +} + +const char *taos_errstr(TAOS_RES *res) { + +} + +void taos_free_result(TAOS_RES *res) { + +} \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c new file mode 100644 index 0000000000000000000000000000000000000000..43d73bf3db85732037e597ab62bf0b0a2830c1dd --- /dev/null +++ b/source/client/src/tscEnv.c @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "tscLog.h" +#include "tsched.h" +#include "ttime.h" +#include "trpc.h" +#include "ttimezone.h" +#include "clientInt.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +SAppInfo appInfo; +int32_t tscReqRef = -1; +int32_t tscConnRef = -1; +void *tscQhandle = NULL; + +int32_t tsNumOfThreads = 1; +volatile int32_t tscInitRes = 0; + +static void registerRequest(SRequestObj* pRequest) { + STscObj *pTscObj = (STscObj *)taosAcquireRef(tscConnRef, pRequest->pTscObj->id); + assert(pTscObj != NULL); + + // connection has been released already, abort creating request. + pRequest->self = taosAddRef(tscReqRef, pRequest); + + int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); + + if (pTscObj->pAppInfo) { + SInstanceActivity *pActivity = &pTscObj->pAppInfo->summary; + + int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d", pRequest->self, + pRequest->pTscObj->id, num, currentInst, total); + } +} + +static void deregisterRequest(SRequestObj* pRequest) { + assert(pRequest != NULL); + + STscObj* pTscObj = pRequest->pTscObj; + SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; + + int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + + tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + taosReleaseRef(tscConnRef, pTscObj->id); +} + +static void tscInitLogFile() { + taosReadGlobalLogCfg(); + if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { + printf("failed to create log dir:%s\n", tsLogDir); + } + + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + +void closeTransporter(STscObj* pTscObj) { + if (pTscObj == NULL || pTscObj->pTransporter == NULL) { + return; + } + + tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id); + rpcClose(pTscObj->pTransporter); + pTscObj->pTransporter = NULL; +} + +// TODO refactor +void* openTransporter(const char *user, const char *auth) { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "TSC"; + rpcInit.numOfThreads = tsNumOfThreads; + rpcInit.cfp = processMsgFromServer; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)user; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; +// rpcInit.spi = 1; + rpcInit.secret = (char *)auth; + + void* pDnodeConn = rpcOpen(&rpcInit); + if (pDnodeConn == NULL) { + tscError("failed to init connection to server"); + return NULL; + } + + return pDnodeConn; +} + +void destroyTscObj(void *pObj) { + STscObj *pTscObj = pObj; + + atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + + closeTransporter(pTscObj); + pthread_mutex_destroy(&pTscObj->mutex); + tfree(pTscObj); +} + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo) { + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); + if (NULL == pObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + pObj->pAppInfo = pAppInfo; + if (pAppInfo != NULL) { + pObj->pTransporter = pAppInfo->pTransporter; + } + + tstrncpy(pObj->user, user, sizeof(pObj->user)); + memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); + + pthread_mutex_init(&pObj->mutex, NULL); + pObj->id = taosAddRef(tscConnRef, pObj); + + tscDebug("connObj created, 0x%"PRIx64, pObj->id); + return pObj; +} + +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { + assert(pObj != NULL); + + SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + if (NULL == pRequest) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + // TODO generated request uuid + pRequest->requestId = 0; + + pRequest->metric.start = taosGetTimestampMs(); + + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; + pRequest->body.param = param; + tsem_init(&pRequest->body.rspSem, 0, 0); + + registerRequest(pRequest); + return pRequest; +} + +static void doDestroyRequest(void* p) { + assert(p != NULL); + SRequestObj* pRequest = (SRequestObj*)p; + + assert(RID_VALID(pRequest->self)); + + tfree(pRequest->msgBuf); + tfree(pRequest->sqlstr); + tfree(pRequest->pInfo); + + deregisterRequest(pRequest); + tfree(pRequest); +} + +void destroyRequest(SRequestObj* pRequest) { + if (pRequest == NULL) { + return; + } + + taosReleaseRef(tscReqRef, pRequest->self); +} + +void taos_init_imp(void) { + // In the APIs of other program language, taos_cleanup is not available yet. + // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. + atexit(taos_cleanup); + + errno = TSDB_CODE_SUCCESS; + srand(taosGetTimestampSec()); + + deltaToUtcInitOnce(); + taosInitGlobalCfg(); + taosReadCfgFromFile(); + + tscInitLogFile(); + if (taosCheckAndPrintCfg()) { + tscInitRes = -1; + return; + } + + taosInitNotes(); + initMsgHandleFp(); + + rpcInit(); + + tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + + taosSetCoreDump(true); + + double factor = 4.0; + int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + + int32_t queueSize = tsMaxConnections * 2; + tscQhandle = taosInitScheduler(queueSize, numOfThreads, "tsc"); + if (NULL == tscQhandle) { + tscError("failed to init task queue"); + tscInitRes = -1; + return; + } + + tscDebug("client task queue is initialized, numOfThreads: %d", numOfThreads); + tscConnRef = taosOpenRef(200, destroyTscObj); + tscReqRef = taosOpenRef(40960, doDestroyRequest); + + taosGetAppName(appInfo.appName, NULL); + appInfo.pid = taosGetPId(); + appInfo.startTime = taosGetTimestampMs(); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + tscDebug("client is initialized successfully"); +} + +int taos_options_imp(TSDB_OPTION option, const char *str) { + SGlobalCfg *cfg = NULL; + + switch (option) { + case TSDB_OPTION_CONFIGDIR: + cfg = taosGetConfigOption("configDir"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(configDir, str, TSDB_FILENAME_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set config file directory:%s", str); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + case TSDB_OPTION_SHELL_ACTIVITY_TIMER: + cfg = taosGetConfigOption("shellActivityTimer"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tsShellActivityTimer = atoi(str); + if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; + if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + } + break; + + case TSDB_OPTION_LOCALE: { // set locale + cfg = taosGetConfigOption("locale"); + assert(cfg != NULL); + + size_t len = strlen(str); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("Invalid locale:%s, use default", str); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + char sep = '.'; + + if (strlen(tsLocale) == 0) { // locale does not set yet + char* defaultLocale = setlocale(LC_CTYPE, ""); + + // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. + // The launch of current system will abort soon. + if (defaultLocale == NULL) { + tscError("failed to get default locale, please set the correct locale in current OS"); + return -1; + } + + tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + } + + // set the user specified locale + char *locale = setlocale(LC_CTYPE, str); + + if (locale != NULL) { // failed to set the user specified locale + tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { // set the user specified locale failed, use default LC_CTYPE as current locale + locale = setlocale(LC_CTYPE, tsLocale); + tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); + } + + tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + + char *charset = strrchr(tsLocale, sep); + if (charset != NULL) { + charset += 1; + + charset = taosCharsetReplace(charset); + + if (taosValidateEncodec(charset)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset set:%s", charset); + } else { + tscInfo("charset changed from %s to %s", tsCharset, charset); + } + + tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + + } else { + tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); + } + + free(charset); + } else { // it may be windows system + tscInfo("charset remains:%s", tsCharset); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + } + + case TSDB_OPTION_CHARSET: { + /* set charset will override the value of charset, assigned during system locale changed */ + cfg = taosGetConfigOption("charset"); + assert(cfg != NULL); + + size_t len = strlen(str); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("failed to set charset:%s", str); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + if (taosValidateEncodec(str)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset is set:%s", str); + } else { + tscInfo("charset changed from %s to %s", tsCharset, str); + } + + tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { + tscInfo("charset:%s not valid", str); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + + break; + } + + case TSDB_OPTION_TIMEZONE: + cfg = taosGetConfigOption("timezone"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); + tsSetTimeZone(); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + default: + // TODO return the correct error code to client in the format for taos_errstr() + tscError("Invalid option %d", option); + return -1; + } + + return 0; +} + +#if 0 +#include "cJSON.h" +static setConfRet taos_set_config_imp(const char *config){ + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + static bool setConfFlag = false; + if (setConfFlag) { + ret.retCode = SET_CONF_RET_ERR_ONLY_ONCE; + strcpy(ret.retMsg, "configuration can only set once"); + return ret; + } + taosInitGlobalCfg(); + cJSON *root = cJSON_Parse(config); + if (root == NULL){ + ret.retCode = SET_CONF_RET_ERR_JSON_PARSE; + strcpy(ret.retMsg, "parse json error"); + return ret; + } + + int size = cJSON_GetArraySize(root); + if(!cJSON_IsObject(root) || size == 0) { + ret.retCode = SET_CONF_RET_ERR_JSON_INVALID; + strcpy(ret.retMsg, "json content is invalid, must be not empty object"); + return ret; + } + + if(size >= 1000) { + ret.retCode = SET_CONF_RET_ERR_TOO_LONG; + strcpy(ret.retMsg, "json object size is too long"); + return ret; + } + + for(int i = 0; i < size; i++){ + cJSON *item = cJSON_GetArrayItem(root, i); + if(!item) { + ret.retCode = SET_CONF_RET_ERR_INNER; + strcpy(ret.retMsg, "inner error"); + return ret; + } + if(!taosReadConfigOption(item->string, item->valuestring, NULL, NULL, TAOS_CFG_CSTATUS_OPTION, TSDB_CFG_CTYPE_B_CLIENT)){ + ret.retCode = SET_CONF_RET_ERR_PART; + if (strlen(ret.retMsg) == 0){ + snprintf(ret.retMsg, RET_MSG_LENGTH, "part error|%s", item->string); + }else{ + int tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + size_t leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, "|", leftSize); + tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, item->string, leftSize); + } + } + } + cJSON_Delete(root); + setConfFlag = true; + return ret; +} + +setConfRet taos_set_config(const char *config){ + pthread_mutex_lock(&setConfMutex); + setConfRet ret = taos_set_config_imp(config); + pthread_mutex_unlock(&setConfMutex); + return ret; +} +#endif \ No newline at end of file diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3f2ad88bb6d8fc35e17ec76dd18539655a607fd --- /dev/null +++ b/source/client/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build parser unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(clientTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + clientTest + PUBLIC os util common transport gtest taos +) + +TARGET_INCLUDE_DIRECTORIES( + clientTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc" +) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8da4caf7eb53c58dd29a4d4b605ff9b2e1089ee9 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "tglobal.h" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "taos.h" + +namespace { +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, driverInit_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + taos_close(pConn); +} \ No newline at end of file diff --git a/source/common/inc/commonInt.h b/source/common/inc/commonInt.h index b8be8899f349474aa4decd2ecb4d59701d32132e..448600e22e0ab525dcd143d485603cee589b9a4b 100644 --- a/source/common/inc/commonInt.h +++ b/source/common/inc/commonInt.h @@ -20,8 +20,21 @@ extern "C" { #endif + +#include "tlog.h" + +extern int32_t cDebugFlag; + +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) + #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_INT_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_INT_H_*/ diff --git a/source/common/src/taosmsg.c b/source/common/src/taosmsg.c new file mode 100644 index 0000000000000000000000000000000000000000..b35e3f1478fde0cfc75243bba74f6a35bec629ad --- /dev/null +++ b/source/common/src/taosmsg.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define TAOS_MESSAGE_C + +#include "taosmsg.h" + + diff --git a/source/common/src/tep.c b/source/common/src/tep.c new file mode 100644 index 0000000000000000000000000000000000000000..ef47a0e0c2a5abc0a21692bd88fb1030ecf07de8 --- /dev/null +++ b/source/common/src/tep.c @@ -0,0 +1,41 @@ +#include "tep.h" +#include "tglobal.h" +#include "tlockfree.h" + +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { + *port = 0; + strcpy(fqdn, ep); + + char *temp = strchr(fqdn, ':'); + if (temp) { + *temp = 0; + *port = atoi(temp+1); + } + + if (*port == 0) { + *port = tsServerPort; + return -1; + } + + return 0; +} + +bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { + return false; + } + + for (int32_t i = 0; i < s1->numOfEps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} + +void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet) { + taosCorBeginWrite(&pEpSet->version); + pEpSet->epSet = *pNewEpSet; + taosCorEndWrite(&pEpSet->version); +} + diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7e71c6bfc7fec62ced2ff3ab763e7d0f66fddd14..dfbbe33b59850ccc1bb29dffbbcaf4027c434a8b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -25,6 +25,7 @@ #include "tutil.h" #include "ttimezone.h" #include "tlocale.h" +#include "tep.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -161,16 +162,8 @@ int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; // restful -int8_t tsEnableHttpModule = 1; int32_t tsRestRowLimit = 10240; -uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041] -int32_t tsHttpCacheSessions = 1000; -int32_t tsHttpSessionExpire = 36000; -int32_t tsHttpMaxThreads = 2; -int8_t tsHttpEnableCompress = 1; -int8_t tsHttpEnableRecordSql = 0; int8_t tsTelegrafUseFieldNum = 0; -int8_t tsHttpDbNameMandatory = 0; // mqtt int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default @@ -193,7 +186,6 @@ int8_t tsEnableStream = 1; // internal int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; -int8_t tscEmbedded = 0; char tsVnodeDir[PATH_MAX] = {0}; char tsDnodeDir[PATH_MAX] = {0}; char tsMnodeDir[PATH_MAX] = {0}; @@ -256,7 +248,6 @@ void taosSetAllDebugFlag() { sdbDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; - cDebugFlag = debugFlag; jniDebugFlag = debugFlag; odbcDebugFlag = debugFlag; httpDebugFlag = debugFlag; @@ -330,7 +321,7 @@ int32_t taosCfgDynamicOptions(char *msg) { if (strncasecmp(option, "resetlog", 8) == 0) { taosResetLog(); - taosPrintGlobalCfg(); + taosPrintCfg(); return 0; } @@ -394,7 +385,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "secondEp"; cfg.ptr = tsSecond; @@ -404,7 +395,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fqdn"; cfg.ptr = tsLocalFqdn; @@ -414,7 +405,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FQDN_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // port cfg.option = "serverPort"; @@ -425,7 +416,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65056; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // directory cfg.option = "configDir"; @@ -436,7 +427,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "logDir"; cfg.ptr = tsLogDir; @@ -446,7 +437,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "scriptDir"; cfg.ptr = tsScriptDir; @@ -456,7 +447,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dataDir"; cfg.ptr = tsDataDir; @@ -466,7 +457,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "arbitrator"; cfg.ptr = tsArbitrator; @@ -476,7 +467,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // dnode configs cfg.option = "numOfThreadsPerCore"; @@ -487,7 +478,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "numOfCommitThreads"; cfg.ptr = &tsNumOfCommitThreads; @@ -497,7 +488,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "ratioOfQueryCores"; cfg.ptr = &tsRatioOfQueryCores; @@ -507,7 +498,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxNumOfDistinctRes"; cfg.ptr = &tsMaxNumOfDistinctResults; @@ -517,7 +508,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000*10000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "numOfMnodes"; cfg.ptr = &tsNumOfMnodes; @@ -527,7 +518,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 3; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "vnodeBak"; cfg.ptr = &tsEnableVnodeBak; @@ -537,7 +528,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; @@ -547,7 +538,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "balance"; cfg.ptr = &tsEnableBalance; @@ -557,7 +548,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "balanceInterval"; cfg.ptr = &tsBalanceInterval; @@ -567,7 +558,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 30000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // 0-any; 1-mnode; 2-vnode cfg.option = "role"; @@ -578,7 +569,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // timer cfg.option = "maxTmrCtrl"; @@ -589,7 +580,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2048; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monitorInterval"; cfg.ptr = &tsMonitorInterval; @@ -599,7 +590,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 600; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "offlineThreshold"; cfg.ptr = &tsOfflineThreshold; @@ -609,7 +600,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 86400 * 365; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcTimer"; cfg.ptr = &tsRpcTimer; @@ -619,7 +610,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 3000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcForceTcp"; cfg.ptr = &tsRpcForceTcp; @@ -629,7 +620,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcMaxTime"; cfg.ptr = &tsRpcMaxTime; @@ -639,7 +630,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 7200; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; @@ -649,7 +640,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "shellActivityTimer"; cfg.ptr = &tsShellActivityTimer; @@ -659,7 +650,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 120; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; @@ -669,7 +660,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minIntervalTime"; cfg.ptr = &tsMinIntervalTime; @@ -679,7 +670,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxStreamCompDelay"; cfg.ptr = &tsMaxStreamComputDelay; @@ -689,7 +680,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxFirstStreamCompDelay"; cfg.ptr = &tsStreamCompStartDelay; @@ -699,7 +690,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "retryStreamCompDelay"; cfg.ptr = &tsRetryStreamCompDelay; @@ -710,7 +701,7 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "streamCompDelayRatio"; cfg.ptr = &tsStreamComputDelayRatio; cfg.valType = TAOS_CFG_VTYPE_FLOAT; @@ -719,7 +710,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0.9f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxVgroupsPerDb"; cfg.ptr = &tsMaxVgroupsPerDb; @@ -729,7 +720,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 8192; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cache"; cfg.ptr = &tsCacheBlockSize; @@ -739,7 +730,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "blocks"; cfg.ptr = &tsBlocksPerVnode; @@ -749,7 +740,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "days"; cfg.ptr = &tsDaysPerFile; @@ -759,7 +750,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "keep"; cfg.ptr = &tsDaysToKeep; @@ -769,7 +760,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_KEEP; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minRows"; cfg.ptr = &tsMinRowsInFileBlock; @@ -779,7 +770,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRows"; cfg.ptr = &tsMaxRowsInFileBlock; @@ -789,7 +780,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "precision"; cfg.ptr = &tsTimePrecision; @@ -799,7 +790,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_PRECISION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "comp"; cfg.ptr = &tsCompression; @@ -809,7 +800,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_COMP_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "walLevel"; cfg.ptr = &tsWAL; @@ -819,7 +810,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_WAL_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fsync"; cfg.ptr = &tsFsyncPeriod; @@ -829,7 +820,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FSYNC_PERIOD; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "replica"; cfg.ptr = &tsReplications; @@ -839,7 +830,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "quorum"; cfg.ptr = &tsQuorum; @@ -849,7 +840,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "update"; cfg.ptr = &tsUpdate; @@ -859,7 +850,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_UPDATE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cachelast"; cfg.ptr = &tsCacheLastRow; @@ -869,7 +860,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_CACHE_LAST_ROW; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttHostName"; cfg.ptr = tsMqttHostName; @@ -879,7 +870,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_HOSTNAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttPort"; cfg.ptr = tsMqttPort; @@ -889,7 +880,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_PORT_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttTopic"; cfg.ptr = tsMqttTopic; @@ -899,7 +890,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_TOPIC_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "compressMsgSize"; cfg.ptr = &tsCompressMsgSize; @@ -909,7 +900,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "compressColData"; cfg.ptr = &tsCompressColData; @@ -919,7 +910,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxSQLLength"; cfg.ptr = &tsMaxSQLStringLen; @@ -929,7 +920,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxWildCardsLength"; cfg.ptr = &tsMaxWildCardsLen; @@ -939,7 +930,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRegexStringLen"; cfg.ptr = &tsMaxRegexStringLen; @@ -949,7 +940,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxNumOfOrderedRes"; cfg.ptr = &tsMaxNumOfOrderedResults; @@ -959,7 +950,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "queryBufferSize"; cfg.ptr = &tsQueryBufferSize; @@ -969,7 +960,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 500000000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "retrieveBlockingModel"; cfg.ptr = &tsRetrieveBlockingModel; @@ -979,7 +970,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "keepColumnName"; cfg.ptr = &tsKeepOriginalColumnName; @@ -989,7 +980,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // locale & charset cfg.option = "timezone"; @@ -1000,7 +991,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_TIMEZONE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "locale"; cfg.ptr = tsLocale; @@ -1010,7 +1001,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "charset"; cfg.ptr = tsCharset; @@ -1020,7 +1011,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // connect configs cfg.option = "maxShellConns"; @@ -1031,7 +1022,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 50000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxConnections"; cfg.ptr = &tsMaxConnections; @@ -1041,7 +1032,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalLogDirGB"; cfg.ptr = &tsMinimalLogDirGB; @@ -1051,7 +1042,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalTmpDirGB"; cfg.ptr = &tsReservedTmpDirectorySpace; @@ -1061,7 +1052,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalDataDirGB"; cfg.ptr = &tsMinimalDataDirGB; @@ -1071,7 +1062,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // module configs cfg.option = "flowctrl"; @@ -1082,7 +1073,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "slaveQuery"; cfg.ptr = &tsEnableSlaveQuery; @@ -1092,7 +1083,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "adjustMaster"; cfg.ptr = &tsEnableAdjustMaster; @@ -1102,17 +1093,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "http"; - cfg.ptr = &tsEnableHttpModule; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqtt"; cfg.ptr = &tsEnableMqttModule; @@ -1122,7 +1103,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monitor"; cfg.ptr = &tsEnableMonitorModule; @@ -1132,7 +1113,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "stream"; cfg.ptr = &tsEnableStream; @@ -1142,7 +1123,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "topicBianryLen"; cfg.ptr = &tsTopicBianryLen; @@ -1152,17 +1133,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 16000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpEnableRecordSql"; - cfg.ptr = &tsHttpEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "telegrafUseFieldNum"; cfg.ptr = &tsTelegrafUseFieldNum; @@ -1172,17 +1143,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpMaxThreads"; - cfg.ptr = &tsHttpMaxThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 2; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "restfulRowLimit"; cfg.ptr = &tsRestRowLimit; @@ -1192,17 +1153,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpDbNameMandatory"; - cfg.ptr = &tsHttpDbNameMandatory; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // debug flag cfg.option = "numOfLogLines"; @@ -1213,7 +1164,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "logKeepDays"; cfg.ptr = &tsLogKeepDays; @@ -1223,7 +1174,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 365000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; @@ -1233,7 +1184,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "debugFlag"; cfg.ptr = &debugFlag; @@ -1243,7 +1194,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mDebugFlag"; cfg.ptr = &mDebugFlag; @@ -1253,7 +1204,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dDebugFlag"; cfg.ptr = &dDebugFlag; @@ -1263,7 +1214,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "sDebugFlag"; cfg.ptr = &sDebugFlag; @@ -1273,7 +1224,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "wDebugFlag"; cfg.ptr = &wDebugFlag; @@ -1283,7 +1234,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "sdbDebugFlag"; @@ -1294,7 +1245,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcDebugFlag"; cfg.ptr = &rpcDebugFlag; @@ -1304,7 +1255,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tmrDebugFlag"; cfg.ptr = &tmrDebugFlag; @@ -1314,7 +1265,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cDebugFlag"; cfg.ptr = &cDebugFlag; @@ -1324,7 +1275,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "jniDebugFlag"; cfg.ptr = &jniDebugFlag; @@ -1334,7 +1285,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "odbcDebugFlag"; cfg.ptr = &odbcDebugFlag; @@ -1344,7 +1295,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "uDebugFlag"; cfg.ptr = &uDebugFlag; @@ -1354,7 +1305,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "httpDebugFlag"; cfg.ptr = &httpDebugFlag; @@ -1364,7 +1315,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttDebugFlag"; cfg.ptr = &mqttDebugFlag; @@ -1374,7 +1325,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monDebugFlag"; cfg.ptr = &monDebugFlag; @@ -1384,7 +1335,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "qDebugFlag"; cfg.ptr = &qDebugFlag; @@ -1394,7 +1345,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "vDebugFlag"; cfg.ptr = &vDebugFlag; @@ -1404,7 +1355,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tsdbDebugFlag"; cfg.ptr = &tsdbDebugFlag; @@ -1414,7 +1365,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cqDebugFlag"; cfg.ptr = &cqDebugFlag; @@ -1424,7 +1375,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; @@ -1434,7 +1385,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "enableCoreFile"; cfg.ptr = &tsEnableCoreFile; @@ -1444,7 +1395,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // version info cfg.option = "gitinfo"; @@ -1455,7 +1406,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "gitinfoOfInternal"; cfg.ptr = gitinfoOfInternal; @@ -1465,7 +1416,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "buildinfo"; cfg.ptr = buildinfo; @@ -1475,7 +1426,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "version"; cfg.ptr = version; @@ -1485,7 +1436,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxBinaryDisplayWidth"; cfg.ptr = &tsMaxBinaryDisplayWidth; @@ -1495,7 +1446,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tempDir"; cfg.ptr = tsTempDir; @@ -1505,7 +1456,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = PATH_MAX; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tsdbMetaCompactRatio"; cfg.ptr = &tsTsdbMetaCompactRatio; @@ -1515,7 +1466,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // enable kill long query cfg.option = "deadLockKillQuery"; @@ -1526,7 +1477,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); #ifdef TD_TSZ // lossy compress @@ -1538,7 +1489,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = tListLen(lossyColumns); cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fPrecision"; cfg.ptr = &fPrecision; @@ -1550,7 +1501,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dPrecision"; cfg.ptr = &dPrecision; @@ -1560,7 +1511,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = MAX_FLOAT; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRange"; cfg.ptr = &maxRange; @@ -1570,7 +1521,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "range"; cfg.ptr = &curRange; @@ -1580,7 +1531,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); #else //assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); @@ -1592,7 +1543,7 @@ void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -int32_t taosCheckGlobalCfg() { +int32_t taosCheckAndPrintCfg() { char fqdn[TSDB_FQDN_LEN]; uint16_t port; @@ -1640,37 +1591,13 @@ int32_t taosCheckGlobalCfg() { tsNumOfCores = 1; } - if (tsHttpMaxThreads == 2) { - int32_t halfNumOfCores = tsNumOfCores >> 1; - if (halfNumOfCores > 2) { - tsHttpMaxThreads = halfNumOfCores; - } - } - - // todo refactor - tsVersion = 0; - for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) { - if (version[i] >= '0' && version[i] <= '9') { - ver = ver * 10 + (version[i] - '0'); - } else if (version[i] == '.') { - tsVersion |= ver & 0xFF; - tsVersion <<= 8; - - ver = 0; - } else if (version[i] == 0) { - tsVersion |= ver & 0xFF; - - break; - } - } - if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } uInfo(" check global cfg completed"); uInfo("=================================="); - taosPrintGlobalCfg(); + taosPrintCfg(); return 0; } @@ -1703,3 +1630,4 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *d return true; } + \ No newline at end of file diff --git a/source/common/src/tmessage.c b/source/common/src/tmessage.c index 0b6dbfdb51baa45bb968f7281514ad039ae41434..07adf1d599469fa6b0bb9dad4e1f0348849c68fa 100644 --- a/source/common/src/tmessage.c +++ b/source/common/src/tmessage.c @@ -13,6 +13,181 @@ * along with this program. If not, see . */ -#define TAOS_MESSAGE_C - #include "taosmsg.h" +#include "commonint.h" + + +int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; + +int32_t (*tscProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; + + +int32_t tscBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + *msgLen = 0; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == input || NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SBuildTableMetaInput* bInput = (SBuildTableMetaInput *)input; + + int32_t estimateSize = sizeof(STableInfoMsg); + if (NULL == *msg || msgSize < estimateSize) { + tfree(*msg); + *msg = calloc(1, estimateSize); + if (NULL == *msg) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + STableInfoMsg *bMsg = (STableInfoMsg *)*msg; + + bMsg->msgHead.vgId = bInput->vgId; + + strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); + bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; + + *msgLen = (int32_t)sizeof(*bMsg); + + return TSDB_CODE_SUCCESS; +} + + +int32_t tscProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SVgroupListRspMsg *pRsp = (SVgroupListRspMsg *)msg; + + pRsp->vgroupNum = htonl(pRsp->vgroupNum); + pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); + + if (pRsp->vgroupNum < 0) { + tscError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + if (pRsp->vgroupVersion < 0) { + tscError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + if (msgSize != (pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp))) { + tscError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + // keep SVgroupListInfo/SVgroupListRspMsg the same + *(SVgroupListInfo **)output = (SVgroupListInfo *)msg; + + if (pRsp->vgroupNum == 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pRsp->vgroupNum; ++i) { + pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); + for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { + pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); + } + } + + return TSDB_CODE_SUCCESS; +} + +void msgInit() { + tscBuildMsg[TSDB_MSG_TYPE_TABLE_META] = tscBuildTableMetaReqMsg; + tscBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = tscBuildVgroupListReqMsg; + + + //tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = tscProcessVgroupListRsp; + +/* + tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; + tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; + tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; + + tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; + tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; + + tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; + tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; + + tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; + tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; + tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; + tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; + tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; + tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; + tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; + tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; + tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; + + tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; + + tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; + tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; + + tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; + + tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; + tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; + tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; + tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; + tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; + + tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; + + tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; + tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; + tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; +*/ +} + + + + + diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 82201df072eba14c11a351c30a99b07d372c37d4..0a725d9975c4a9379825169a7d895366fb2d7a5f 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -112,12 +112,12 @@ int dmnReadConfig(const char *path) { return -1; } - if (taosReadGlobalCfg() != 0) { + if (taosReadCfgFromFile() != 0) { uError("failed to read global config"); return -1; } - if (taosCheckGlobalCfg() != 0) { + if (taosCheckAndPrintCfg() != 0) { uError("failed to check global config"); return -1; } diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 130cbbef07fff3a7b0be12fa9d3bb7a48eab1c7d..9e2d4d4c9b00417ce0ccf91042c93d8e41a1f962 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -17,6 +17,7 @@ #include "dndDnode.h" #include "dndTransport.h" #include "dndVnodes.h" +#include "tep.h" int32_t dndGetDnodeId(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 2ec02ac2030b730e5f6a539bb758453b4e5c8a4b..6dc46cefcdd4260df881d57126ae7773d98964b9 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -72,7 +72,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_STB_VGROUP] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_VGROUP_LIST] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; @@ -235,18 +235,18 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp static int32_t dndAuthInternalMsg(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, INTERNAL_USER) == 0) { // A simple temporary implementation - char pass[32] = {0}; + char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - memcpy(secret, pass, TSDB_KEY_LEN); + memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 0; *encrypt = 0; *ckey = 0; return 0; } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { // A simple temporary implementation - char pass[32] = {0}; + char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - memcpy(secret, pass, TSDB_KEY_LEN); + memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 0; *encrypt = 0; *ckey = 0; @@ -288,8 +288,8 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); } else { SAuthRsp *pRsp = rpcRsp.pCont; - memcpy(secret, pRsp->secret, TSDB_KEY_LEN); - memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); + memcpy(secret, pRsp->secret, TSDB_PASSWORD_LEN); + memcpy(ckey, pRsp->ckey, TSDB_PASSWORD_LEN); *spi = pRsp->spi; *encrypt = pRsp->encrypt; dDebug("user:%s, success to get user auth from other mnodes", user); @@ -368,4 +368,4 @@ void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pMsg) { SEpSet epSet = {0}; dndGetMnodeEpSet(pDnode, &epSet); dndSendMsgToDnode(pDnode, &epSet, pMsg); -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 61fc27a59595b905c4e28c6311e5e1d6a40f792b..de50899c2d3c121f3ef34e764be5352cab91ca90 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -120,7 +120,7 @@ SClient* createClient(const char* user, const char* pass, const char* fqdn, uint SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); ASSERT(pClient); - char secretEncrypt[32] = {0}; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index aace3f4ee3cb9a15342bd175ec98abccddfed8fa..5ddde3181eb64fa0ddee76a9daeac13b6fd5abda 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -180,13 +180,11 @@ typedef struct SAcctObj { typedef struct SUserObj { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; - int8_t superAuth; - int8_t readAuth; - int8_t writeAuth; + int8_t superUser; int32_t acctId; SHashObj *prohibitDbHash; } SUserObj; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index b4298797895718b5484f5837e72340811594e43d..59d84e5760206d8a331c23ab3a99dd3490d301e5 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -29,6 +29,7 @@ typedef struct { char user[TSDB_USER_LEN]; char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc int32_t pid; // pid of app that invokes taosc + int64_t appStartTime; // app start time int32_t id; int8_t killed; int8_t align; @@ -44,7 +45,7 @@ typedef struct { SQueryDesc *pQueries; } SConnObj; -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app); +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); @@ -102,13 +103,14 @@ void mndCleanupProfile(SMnode *pMnode) { } } -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app) { +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); SConnObj connObj = {.pid = pid, + .appStartTime = startTime, .id = connId, .killed = 0, .port = port, @@ -195,6 +197,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SConnectMsg *pReq = pMsg->rpcMsg.pCont; pReq->pid = htonl(pReq->pid); + pReq->startTime = htobe64(pReq->startTime); SRpcConnInfo info = {0}; if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { @@ -216,7 +219,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); } - SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, pReq->startTime); if (pConn == NULL) { mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); return -1; @@ -233,9 +236,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser != NULL) { pRsp->acctId = htonl(pUser->acctId); - pRsp->superAuth = pUser->superAuth; - pRsp->readAuth = pUser->readAuth; - pRsp->writeAuth = pUser->writeAuth; + pRsp->superUser = pUser->superUser; mndReleaseUser(pMnode, pUser); } @@ -246,7 +247,8 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { pMsg->contLen = sizeof(SConnectRsp); pMsg->pCont = pRsp; - mDebug("user:%s, login from %s, conn:%d", info.user, ip, pConn->id); + + mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pReq->app); return 0; } @@ -301,7 +303,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId); if (pConn == NULL) { - pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, 0); if (pConn == NULL) { mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); return -1; @@ -368,7 +370,7 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -399,7 +401,7 @@ static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -430,7 +432,7 @@ static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -459,7 +461,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -587,7 +589,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -803,7 +805,7 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2f582a810d8a1140fb0dcbc1eb482bfcea364cc1..2111047219a850a289c82cf5a1246c136b0c8cc3 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -65,11 +65,9 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.readAuth = 1; - userObj.writeAuth = 1; if (strcmp(user, TSDB_DEFAULT_USER) == 0) { - userObj.superAuth = 1; + userObj.superUser = 1; } SSdbRaw *pRaw = mndUserActionEncode(&userObj); @@ -89,7 +87,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { return -1; } -#endif +#endif return 0; } @@ -100,13 +98,11 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t dataPos = 0; SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN) - SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_KEY_LEN) + SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN) SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) - SDB_SET_INT8(pRaw, dataPos, pUser->superAuth) - SDB_SET_INT8(pRaw, dataPos, pUser->readAuth) - SDB_SET_INT8(pRaw, dataPos, pUser->writeAuth) + SDB_SET_INT8(pRaw, dataPos, pUser->superUser) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -128,13 +124,11 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->user, TSDB_USER_LEN) - SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->pass, TSDB_KEY_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->pass, TSDB_PASSWORD_LEN) SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->acct, TSDB_USER_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superAuth) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->readAuth) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->writeAuth) + SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superUser) return pRow; } @@ -173,13 +167,11 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) { mTrace("user:%s, perform update action", pOldUser->user); memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN); - memcpy(pOldUser->pass, pNewUser->pass, TSDB_KEY_LEN); + memcpy(pOldUser->pass, pNewUser->pass, TSDB_PASSWORD_LEN); memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN); pOldUser->createdTime = pNewUser->createdTime; pOldUser->updateTime = pNewUser->updateTime; - pOldUser->superAuth = pNewUser->superAuth; - pOldUser->readAuth = pNewUser->readAuth; - pOldUser->writeAuth = pNewUser->writeAuth; + pOldUser->superUser = pNewUser->superUser; return 0; } @@ -200,9 +192,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superAuth = 0; - userObj.readAuth = 1; - userObj.writeAuth = 1; + userObj.superUser = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { @@ -515,7 +505,7 @@ static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pUser->superAuth) { + if (pUser->superUser) { const char *src = "super"; STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); } else { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5b50bbff4ccde745697d68223d2299b94f0bc41b..720f197782f7ee59dbe525d9cc9d1f5733dde71c 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -21,14 +21,58 @@ extern "C" { #endif #include "catalog.h" +#include "common.h" +#include "tlog.h" + +#define CTG_DEFAULT_CLUSTER_NUMBER 6 +#define CTG_DEFAULT_VGROUP_NUMBER 100 + +#define CTG_DEFAULT_INVALID_VERSION (-1) + +typedef struct SVgroupListCache { + int32_t vgroupVersion; + SHashObj *cache; // key:vgId, value:SVgroupInfo* + SArray *arrayCache; // SVgroupInfo +} SVgroupListCache; + +typedef struct SDBVgroupCache { + SHashObj *cache; //key:dbname, value:SDBVgroupInfo +} SDBVgroupCache; + +typedef struct STableMetaCache { + SHashObj *cache; //key:fulltablename, value:STableMeta +} STableMetaCache; typedef struct SCatalog { - void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata - SHashObj *pData; // items cached for each cluster, the hash key is the cluster-id, returned by mgmt node + SVgroupListCache vgroupCache; + SDBVgroupCache dbCache; + STableMetaCache tableCache; } SCatalog; +typedef struct SCatalogMgmt { + void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata + SHashObj *pCluster; // items cached for each cluster, the hash key is the cluster-id got from mgmt node +} SCatalogMgmt; + + +extern int32_t ctgDebugFlag; + +#define ctgFatal(...) do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgError(...) do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgWarn(...) do { if (ctgDebugFlag & DEBUG_WARN) { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgInfo(...) do { if (ctgDebugFlag & DEBUG_INFO) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgDebug(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) + + +#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) +#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); return _code; } } while (0) +#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) + + #ifdef __cplusplus } #endif -#endif /*_TD_CATALOG_INT_H_*/ \ No newline at end of file +#endif /*_TD_CATALOG_INT_H_*/ diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 08e2172adbff44f8cb2dfc6201f06855578c37e6..f4a4b01dab19a022722727a6f95a07d51a169f1e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -14,11 +14,294 @@ */ #include "catalogInt.h" +#include "trpc.h" +#include "tmessage.h" -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { - return (struct SCatalog*) 0x1; +SCatalogMgmt ctgMgmt = {0}; + +int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SVgroupListInfo** pVgroup) { + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + int32_t code = tscBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_VGROUP_LIST, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + code = tscProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + return code; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* exist) { + if (NULL == pCatalog->vgroupCache.arrayCache || pCatalog->vgroupCache.vgroupVersion < 0) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + if (pVgroupList) { + *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache); + } + + *exist = 1; + + return TSDB_CODE_SUCCESS; +} + + + + + +int32_t catalogInit(SCatalogCfg *cfg) { + ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == ctgMgmt.pCluster) { + CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { + if (NULL == clusterId || NULL == catalogHandle) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + if (NULL == ctgMgmt.pCluster) { + ctgError("cluster cache are not ready"); + return TSDB_CODE_CTG_NOT_READY; + } + + size_t clen = strlen(clusterId); + SCatalog *clusterCtg = (SCatalog *)taosHashGet(ctgMgmt.pCluster, clusterId, clen); + + if (clusterCtg) { + *catalogHandle = clusterCtg; + return TSDB_CODE_SUCCESS; + } + + clusterCtg = calloc(1, sizeof(*clusterCtg)); + if (NULL == clusterCtg) { + ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg)); + return TSDB_CODE_CTG_MEM_ERROR; + } + + clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; + + if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { + ctgError("put cluster %s cache to hash failed", clusterId); + tfree(clusterCtg); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + *catalogHandle = clusterCtg; + + return TSDB_CODE_SUCCESS; } -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { + +int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) { + if (NULL == pCatalog || NULL == version) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + *version = pCatalog->vgroupCache.vgroupVersion; + + return TSDB_CODE_SUCCESS; +} + + + +int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { + if (NULL == pVgroup) { + ctgError("vgroup get from mnode succeed, but no output"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (pVgroup->vgroupVersion < 0) { + ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion); + return TSDB_CODE_CTG_INVALID_INPUT; + } + + + if (NULL == pCatalog->vgroupCache.arrayCache) { + pCatalog->vgroupCache.arrayCache = taosArrayInit(pVgroup->vgroupNum, sizeof(pVgroup->vgroupInfo[0])); + if (NULL == pCatalog->vgroupCache.arrayCache) { + ctgError("init array[%d] for cluster cache failed", pVgroup->vgroupNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } else { + taosArrayClear(pCatalog->vgroupCache.arrayCache); + } + + if (NULL == pCatalog->vgroupCache.cache) { + pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->vgroupCache.cache) { + ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_VGROUP_NUMBER); + return TSDB_CODE_CTG_MEM_ERROR; + } + } else { + taosHashClear(pCatalog->vgroupCache.cache); + } + + SVgroupInfo *vInfo = NULL; + for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) { + vInfo = taosArrayPush(pCatalog->vgroupCache.arrayCache, &pVgroup->vgroupInfo[i]); + if (NULL == vInfo) { + ctgError("push to vgroup array cache failed"); + goto error_exit; + } + + if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &vInfo, POINTER_BYTES) != 0) { + ctgError("push to vgroup hash cache failed"); + goto error_exit; + } + } + + pCatalog->vgroupCache.vgroupVersion = pVgroup->vgroupVersion; + + return TSDB_CODE_SUCCESS; + +error_exit: + if (pCatalog->vgroupCache.arrayCache) { + taosArrayDestroy(pCatalog->vgroupCache.arrayCache); + pCatalog->vgroupCache.arrayCache = NULL; + } + + if (pCatalog->vgroupCache.cache) { + taosHashCleanup(pCatalog->vgroupCache.cache); + pCatalog->vgroupCache.cache = NULL; + } + + pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + + +int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + int32_t exist = 0; + + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + + if (exist) { + return TSDB_CODE_SUCCESS; + } + + SVgroupListInfo *pVgroup = NULL; + + CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup)); + + CTG_ERR_RET(catalogUpdateVgroup(pCatalog, pVgroup)); + + if (pVgroupList) { + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + } + + if (0 == exist) { + ctgError("catalog fetched but get from cache failed"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { + if (NULL == pCatalog || NULL == dbName || NULL == version) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + if (NULL == pCatalog->dbCache.cache) { + *version = CTG_DEFAULT_INVALID_VERSION; + return TSDB_CODE_SUCCESS; + } + + SDBVgroupInfo * dbInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); + if (NULL == dbInfo) { + *version = CTG_DEFAULT_INVALID_VERSION; + return TSDB_CODE_SUCCESS; + } + + *version = dbInfo->vgroupVersion; + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) { + +} + +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { + +} + + + +int32_t catalogGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + SBuildTableMetaInput bInput = {0}; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + int32_t code = tscBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_TABLE_META, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta) { + +} + + +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + return 0; } + +void catalogDestroy(void) { + if (ctgMgmt.pCluster) { + taosHashCleanup(ctgMgmt.pCluster); //TBD + ctgMgmt.pCluster = NULL; + } +} + + + diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 47f655d446cd7f9da52ccce86788c191b06b7a73..e3925d34466562eca7b1bc1dd18f59aaaf02c401 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -79,13 +79,13 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf); * @param msgBufLen * @return */ -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen); +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, char* msg, int32_t msgBufLen); /** * Destroy the meta data request structure. * @param pMetaInfo */ -void qParserClearupMetaRequestInfo(SMetaReq* pMetaInfo); +void qParserClearupMetaRequestInfo(SCatalogReq* pMetaInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 7b6c423fb6608085ec43cb726c98a12f83ea3fdd..d2813886b323c41658f07b50d46da7891d0f66d2 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4077,18 +4077,18 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } #endif - SMetaReq req = {0}; + SCatalogReq req = {0}; SMetaData data = {0}; // TODO: check if the qnode info has been cached already - req.qNodeEpset = true; + req.qNodeRequired = true; code = qParserExtractRequestedMetaInfo(pInfo, &req, msgBuf, msgBufLen); if (code != TSDB_CODE_SUCCESS) { return code; } // load the meta data from catalog - code = catalogGetMetaData(pCatalog, &req, &data); + code = catalogGetAllMeta(pCatalog, NULL, &req, &data); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index c6ae149f1fc4665a9ff4de7825895a5d2dde9565..fa59bc6ca70915aacded5814ec71778c52170871 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -178,15 +178,15 @@ static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray return TSDB_CODE_SUCCESS; } -static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SMetaReq* pMetaReq) { +static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SCatalogReq* pMetaReq) { pMetaReq->pTableName = taosArrayInit(4, sizeof(SName)); return buildTableName(pCxt, pStname, pMetaReq->pTableName); } static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { - SMetaReq req; + SCatalogReq req; CHECK_CODE(buildMetaReq(pCxt, pTname, &req)); - CHECK_CODE(catalogGetMetaData(pCxt->pCatalog, &req, &pCxt->meta)); + CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, &pCxt->meta)); //TODO pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0); return TSDB_CODE_SUCCESS; } @@ -861,13 +861,15 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { .pComCxt = pContext, .pSql = pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pCatalog = getCatalogHandle(pContext->pEpSet), + .pCatalog = NULL, .pTableMeta = NULL, .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .totalNum = 0, .pOutput = *pInfo }; + CHECK_CODE(catalogGetHandle(NULL, &context.pCatalog)); //TODO + if (NULL == context.pTableBlockHashObj) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index a8642e2535fd2cbd57a50558a05000bac3ea029e..d167f7ad3cffe70dc1a17b5a3b56be01ae5451ea 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -43,7 +43,12 @@ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - struct SCatalog* pCatalog = getCatalogHandle(NULL); + struct SCatalog* pCatalog = NULL; + int32_t code = catalogGetHandle(NULL, &pCatalog); + if (code) { + return code; + } + return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); } @@ -132,7 +137,7 @@ static void freePtrElem(void* p) { tfree(*(char**)p); } -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen) { +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, char* msg, int32_t msgBufLen) { int32_t code = TSDB_CODE_SUCCESS; SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen}; @@ -189,7 +194,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet return code; } -void qParserClearupMetaRequestInfo(SMetaReq* pMetaReq) { +void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) { if (pMetaReq == NULL) { return; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 7b3239c0ddb7343e4a68bf247640434e86b798a6..28d01b9e66f35d38abca57a7fbe9504e61ee9647 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1448,23 +1448,6 @@ void* vgroupInfoClear(SVgroupsInfo *vgroupList) { return NULL; } -char* serializeTagData(STagData* pTagData, char* pMsg) { - int32_t n = (int32_t) strlen(pTagData->name); - *(int32_t*) pMsg = htonl(n); - pMsg += sizeof(n); - - memcpy(pMsg, pTagData->name, n); - pMsg += n; - - *(int32_t*)pMsg = htonl(pTagData->dataLen); - pMsg += sizeof(int32_t); - - memcpy(pMsg, pTagData->data, pTagData->dataLen); - pMsg += pTagData->dataLen; - - return pMsg; -} - int32_t copyTagData(STagData* dst, const STagData* src) { dst->dataLen = src->dataLen; tstrncpy(dst->name, src->name, tListLen(dst->name)); diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 1f6fd7103059546e04b96e548f5ad7167ba44935..f2b34971efcb1487929fa6274bc6b34001f7053c 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -48,6 +48,6 @@ struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { return mockCatalogService->getCatalogHandle(pMgmtEps); } -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) { return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); } diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h index 32001d19fd32d93e6cbeb8cd0d251d77743d56f5..e60f7727e6804cc6fb6a8959ecb2440745671163 100644 --- a/source/libs/parser/test/mockCatalog.h +++ b/source/libs/parser/test/mockCatalog.h @@ -22,6 +22,6 @@ void generateMetaData(MockCatalogService* mcs); // mock struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData); #endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 46e82566d6838cd031b051e93f9ebc4b9dbd5f69..457f8d88bdb6d54156e1cc8be29e569978350be9 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -87,7 +87,7 @@ public: return (struct SCatalog*)0x01; } - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) const { + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { assert(nullptr != pMetaReq && 1 == taosArrayGetSize(pMetaReq->pTableName)); SName* fullName = (SName*)taosArrayGet(pMetaReq->pTableName, 0); std::unique_ptr table; @@ -248,7 +248,7 @@ struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) co return impl_->getCatalogHandle(pMgmtEps); } -int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) const { +int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); } diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index 79572086a1f2729393c980653152ebf4523029a1..66b439b3e9f46a6cf1d1350d1777e9ee3eee6852 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -50,7 +50,7 @@ public: MockCatalogService(); ~MockCatalogService(); struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const; - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) const; + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const; ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0); void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); void showTables() const; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 2193a44604602940dbd117080ae07c073be94d05..6a402cd6201040799eef253b9f1612fa43f76fe3 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -38,7 +38,7 @@ void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_ strcpy(p->name, name); } -void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq* req) { +void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SCatalogReq* req) { pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo = (STableMetaInfo**)calloc(1, POINTER_BYTES); @@ -80,7 +80,7 @@ void sqlCheck(const char* sql, bool valid) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -117,7 +117,7 @@ TEST(testCase, validateAST_test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -175,7 +175,7 @@ TEST(testCase, function_Test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -221,7 +221,7 @@ TEST(testCase, function_Test2) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -267,7 +267,7 @@ TEST(testCase, function_Test3) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -312,7 +312,7 @@ TEST(testCase, function_Test4) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -360,7 +360,7 @@ TEST(testCase, function_Test5) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -445,7 +445,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -523,7 +523,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -585,7 +585,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -664,7 +664,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index c86e6876642155a7378e514cfa787da811303fef..bb9271a3c80e928eadee8ee2c42bbfe22b6e4dce 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -39,7 +39,7 @@ void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_ strcpy(p->name, name); } -void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { +void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SCatalogReq *req) { pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo = (STableMetaInfo**)calloc(1, POINTER_BYTES); @@ -79,7 +79,7 @@ void generateLogicplan(const char* sql) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -119,7 +119,7 @@ TEST(testCase, planner_test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 032a93cd32ed926f5bce7fbfc65a4c67bbbb9086..23c0aae15f6d3fe2f22dda4046abefc25beb9f69 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -709,7 +709,7 @@ TEST(testCase, extractMeta_test) { ASSERT_EQ(info1.valid, true); char msg[128] = {0}; - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c index 06af8ff6c2294750663e40bbf958980272216950..e49b1d7983ca3eb8ab05e0278d5f0f8c8912c476 100644 --- a/source/libs/sync/src/sync.c +++ b/source/libs/sync/src/sync.c @@ -228,7 +228,7 @@ static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyn } static int syncInitRpcClient(SSyncManager* syncManager) { - char secret[TSDB_KEY_LEN] = "secret"; + char secret[TSDB_PASSWORD_LEN] = "secret"; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "sync-client"; diff --git a/source/libs/transport/inc/rpcLog.h b/source/libs/transport/inc/rpcLog.h index 6c4a281d2c0f0bf85975200c833acbac856dc40c..904680bbe66be06b30b49be727ca522b9b5ed47e 100644 --- a/source/libs/transport/inc/rpcLog.h +++ b/source/libs/transport/inc/rpcLog.h @@ -23,11 +23,10 @@ extern "C" { #include "tlog.h" extern int32_t rpcDebugFlag; -extern int8_t tscEmbedded; -#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} -#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} -#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} +#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", rpcDebugFlag, __VA_ARGS__); }} +#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", rpcDebugFlag, __VA_ARGS__); }} +#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", rpcDebugFlag, __VA_ARGS__); }} #define tInfo(...) { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tDebug(...) { if (rpcDebugFlag & DEBUG_DEBUG) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }} #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }} diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index fb69a74876c31ef41027dd1958c41d378a8fbbe1..2f710f3efdd0d106dc092e546c5c10f7fc504bbc 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -51,8 +51,8 @@ typedef struct { char user[TSDB_UNI_LEN]; // meter ID char spi; // security parameter index char encrypt; // encrypt algorithm - char secret[TSDB_KEY_LEN]; // secret for the link - char ckey[TSDB_KEY_LEN]; // ciphering key + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key void (*cfp)(void *parent, SRpcMsg *, SEpSet *); int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); @@ -97,8 +97,8 @@ typedef struct SRpcConn { char user[TSDB_UNI_LEN]; // user ID for the link char spi; // security parameter index char encrypt; // encryption, 0:1 - char secret[TSDB_KEY_LEN]; // secret for the link - char ckey[TSDB_KEY_LEN]; // ciphering key + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key char secured; // if set to 1, no authentication uint16_t localPort; // for UDP only uint32_t linkUid; // connection unique ID assigned by client @@ -229,8 +229,6 @@ static void rpcInitImp(void) { tsRpcOverhead = sizeof(SRpcReqContext); tsRpcRefId = taosOpenRef(200, rpcFree); - - return 0; } int32_t rpcInit(void) { @@ -413,7 +411,7 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t // for TDengine, all the query, show commands shall have TCP connection char type = pMsg->msgType; if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_SHOW_RETRIEVE - || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_STB_VGROUP + || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_VGROUP_LIST || type == TSDB_MSG_TYPE_TABLES_META || type == TSDB_MSG_TYPE_TABLE_META || type == TSDB_MSG_TYPE_SHOW || type == TSDB_MSG_TYPE_STATUS || type == TSDB_MSG_TYPE_ALTER_TABLE) pContext->connType = RPC_CONN_TCPC; @@ -705,7 +703,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->linkUid = (uint32_t)((int64_t)pConn + taosGetPid() + (int64_t)pConn->tranId); pConn->spi = pRpc->spi; pConn->encrypt = pRpc->encrypt; - if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN); + if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_PASSWORD_LEN); tDebug("%s %p client connection is allocated, uid:0x%x", pRpc->label, pConn, pConn->linkUid); } @@ -1534,9 +1532,9 @@ static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { int ret = -1; tMD5Init(&context); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Update(&context, (uint8_t *)pMsg, msgLen); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Final(&context); if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; @@ -1548,9 +1546,9 @@ static void rpcBuildAuthHead(void *pMsg, int msgLen, void *pAuth, void *pKey) { T_MD5_CTX context; tMD5Init(&context); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Update(&context, (uint8_t *)pMsg, msgLen); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Final(&context); memcpy(pAuth, context.digest, sizeof(context.digest)); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index ec01f7d7fc91e7689dffbf53d266f7e4f652ee29..e546a87326952e8927c752da409dc6cdc812e985 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -33,10 +33,12 @@ typedef struct WalFileInfo { int64_t fileSize; } WalFileInfo; +#pragma pack(push,1) typedef struct WalIdxEntry { int64_t ver; int64_t offset; } WalIdxEntry; +#pragma pack(pop) static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; @@ -78,11 +80,11 @@ static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) { } static inline int walBuildLogName(SWal*pWal, int64_t fileFirstVer, char* buf) { - return sprintf(buf, "%s/%" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); + return sprintf(buf, "%s/%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); } static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { - return sprintf(buf, "%s/%" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); + return sprintf(buf, "%s/%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); } static inline int walValidHeadCksum(SWalHead* pHead) { diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 7fdd17c17b08eb816da18282cb454fbcd3c4ad4d..629451a7223cb0530f775a550151d3aaf07468eb 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -68,9 +68,12 @@ int32_t walInit() { } void walCleanUp() { + int old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); + if(old == 0) { + return; + } walStopThread(); taosCloseRef(tsWal.refSetId); - atomic_store_8(&tsWal.inited, 0); wInfo("wal module is cleaned up"); } @@ -252,9 +255,8 @@ static int32_t walCreateThread() { static void walStopThread() { atomic_store_8(&tsWal.stop, 1); - if (tsWal.thread != NULL && taosCheckPthreadValid(tsWal.thread)) { + if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); - tsWal.thread = NULL; } wDebug("wal thread is stopped"); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 554a5c846b5df84e76d192bb588a51e7e2020ca5..b6aafedea3b4ac9150370d7012407c34a9c7e311 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -21,16 +21,25 @@ SWalReadHandle* walOpenReadHandle(SWal* pWal) { if(pRead == NULL) { return NULL; } - memset(pRead, 0, sizeof(SWalReadHandle)); pRead->pWal = pWal; pRead->readIdxTfd = -1; pRead->readLogTfd = -1; - return NULL; + pRead->curVersion = -1; + pRead->curFileFirstVer = -1; + pRead->capacity = 0; + pRead->status = 0; + pRead->pHead = malloc(sizeof(SWalHead)); + if(pRead->pHead == NULL) { + free(pRead); + return NULL; + } + return pRead; } void walCloseReadHandle(SWalReadHandle *pRead) { tfClose(pRead->readIdxTfd); tfClose(pRead->readLogTfd); + tfree(pRead->pHead); free(pRead); } @@ -47,18 +56,17 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i //seek position int64_t offset = (ver - fileFirstVer) * WAL_IDX_ENTRY_SIZE; code = tfLseek(idxTfd, offset, SEEK_SET); - if(code != 0) { + if(code < 0) { return -1; } WalIdxEntry entry; - code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry)); - if(code != 0) { + if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { return -1; } //TODO:deserialize ASSERT(entry.ver == ver); code = tfLseek(logTfd, entry.offset, SEEK_SET); - if (code != 0) { + if (code < 0) { return -1; } return code; @@ -71,13 +79,13 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { tfClose(pRead->readLogTfd); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); - int logTfd = tfOpenRead(fnameStr); + int64_t logTfd = tfOpenRead(fnameStr); if(logTfd < 0) { return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); - int idxTfd = tfOpenRead(fnameStr); + int64_t idxTfd = tfOpenRead(fnameStr); if(idxTfd < 0) { return -1; } @@ -90,7 +98,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { int code; SWal *pWal = pRead->pWal; - if(ver == pWal->vers.lastVer) { + if(ver == pRead->curVersion) { return 0; } if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { @@ -126,33 +134,41 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int code; //TODO: check wal life if(pRead->curVersion != ver) { - walReadSeekVer(pRead, ver); + code = walReadSeekVer(pRead, ver); + if(code != 0) { + return -1; + } } if(!tfValid(pRead->readLogTfd)) return -1; - if(sizeof(SWalHead) != tfRead(pRead->readLogTfd, &pRead->head, sizeof(SWalHead))) { + code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + if(code != sizeof(SWalHead)) { return -1; } - code = walValidHeadCksum(&pRead->head); + code = walValidHeadCksum(pRead->pHead); if(code != 0) { return -1; } - if(pRead->capacity < pRead->head.head.len) { - void* ptr = realloc(pRead, pRead->head.head.len); + if(pRead->capacity < pRead->pHead->head.len) { + void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); if(ptr == NULL) { return -1; } - pRead = ptr; - pRead->capacity = pRead->head.head.len; + pRead->pHead = ptr; + pRead->capacity = pRead->pHead->head.len; } - if(pRead->head.head.len != tfRead(pRead->readLogTfd, &pRead->head.head.body, pRead->head.head.len)) { + if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } - code = walValidBodyCksum(&pRead->head); + + /*code = walValidBodyCksum(pRead->pHead);*/ + ASSERT(pRead->pHead->head.version == ver); + if(code != 0) { return -1; } + pRead->curVersion++; return 0; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 44e8cec153f7791e7741244c8f1c5d804ef72e24..994b8fc33323a001578bb38b0741a3a7c0eac5cf 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -377,11 +377,12 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i //must truncate explicitly first return -1; } - /*if (!tfValid(pWal->curLogTfd)) return 0;*/ + /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ pthread_mutex_lock(&pWal->mutex); pWal->writeHead.head.version = index; + int64_t offset = walGetCurFileOffset(pWal); pWal->writeHead.head.len = bodyLen; pWal->writeHead.head.msgType = msgType; pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); @@ -393,12 +394,12 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, &body, bodyLen) != bodyLen) { + if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal)); + code = walWriteIndex(pWal, index, offset); if(code != 0) { //TODO return -1; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 504f1ada3f7d1b66135879538a173caef43e1f2a..200bf39c5a087af9edd7cfefea0ed0a5e1c846e6 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -5,6 +5,9 @@ #include "walInt.h" +const char* ranStr = "tvapq02tcp"; +const int ranStrLen = strlen(ranStr); + class WalCleanEnv : public ::testing::Test { protected: static void SetUpTestCase() { @@ -157,15 +160,13 @@ TEST_F(WalCleanEnv, removeOldMeta) { TEST_F(WalKeepEnv, readOldMeta) { walResetEnv(); - const char* ranStr = "tvapq02tcp"; - int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, len); + code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -179,7 +180,7 @@ TEST_F(WalKeepEnv, readOldMeta) { char* newss = walMetaSerialize(pWal); - len = strlen(oldss); + int len = strlen(oldss); ASSERT_EQ(len, strlen(newss)); for(int i = 0; i < len; i++) { EXPECT_EQ(oldss[i], newss[i]); @@ -189,14 +190,12 @@ TEST_F(WalKeepEnv, readOldMeta) { } TEST_F(WalCleanEnv, write) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, len); + code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -205,11 +204,9 @@ TEST_F(WalCleanEnv, write) { } TEST_F(WalCleanEnv, rollback) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -224,12 +221,10 @@ TEST_F(WalCleanEnv, rollback) { } TEST_F(WalCleanDeleteEnv, roll) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; int i; for(i = 0; i < 100; i++) { - code = walWrite(pWal, i, 0, (void*)ranStr, len); + code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); code = walCommit(pWal, i); @@ -242,19 +237,55 @@ TEST_F(WalCleanDeleteEnv, roll) { ASSERT_EQ(pWal->vers.snapshotVer, i-1); ASSERT_EQ(pWal->vers.verInSnapshotting, -1); - code = walWrite(pWal, 5, 0, (void*)ranStr, len); + code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen); ASSERT_NE(code, 0); for(; i < 200; i++) { - code = walWrite(pWal, i, 0, (void*)ranStr, len); + code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); code = walCommit(pWal, i); ASSERT_EQ(pWal->vers.commitVer, i); } - //code = walWriteMeta(pWal); code = walBeginTakeSnapshot(pWal, i - 1); ASSERT_EQ(code, 0); code = walEndTakeSnapshot(pWal); ASSERT_EQ(code, 0); } + +TEST_F(WalKeepEnv, readHandleRead) { + walResetEnv(); + int code; + SWalReadHandle* pRead = walOpenReadHandle(pWal); + ASSERT(pRead != NULL); + + int i ; + for(i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walWrite(pWal, i, 0, newStr, len); + ASSERT_EQ(code, 0); + } + for(int i = 0; i < 1000; i++) { + int ver = rand() % 100; + code = walReadWithHandle(pRead, ver); + ASSERT_EQ(code, 0); + + //printf("rrbody: \n"); + //for(int i = 0; i < pRead->pHead->head.len; i++) { + //printf("%d ", pRead->pHead->head.body[i]); + //} + //printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver+1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.len, len); + for(int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } +} diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 2385d102858393ef426ac54d4999c413a3f639a5..0d7066b5c8eac3bc7e312651ef0b1888410d62f2 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -42,7 +42,7 @@ bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == s int32_t taosGetPId() { return GetCurrentProcessId(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { +int32_t taosGetAppName(char* name, int32_t* len) { char filepath[1024] = {0}; GetModuleFileName(NULL, filepath, MAX_PATH); @@ -358,7 +358,7 @@ bool taosComparePthread(pthread_t first, pthread_t second) { return pthread_equa int32_t taosGetPId() { return (int32_t)getpid(); } -int32_t taosGetCurrentAPPName(char *name, int32_t *len) { +int32_t taosGetAppName(char *name, int32_t *len) { char buf[PATH_MAX + 1]; buf[0] = '\0'; proc_name(getpid(), buf, sizeof(buf) - 1); @@ -392,7 +392,7 @@ void taosResetPthread(pthread_t* thread) { *thread = 0; } bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } int32_t taosGetPId() { return getpid(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { +int32_t taosGetAppName(char* name, int32_t* len) { const char* self = "/proc/self/exe"; char path[PATH_MAX] = {0}; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 6063f816821ffc869a337e5c01b9e16bfa12efe5..f714fd0dc4d26cf9f5c1391b34c6277b971cd74d 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -275,13 +275,13 @@ char *strsep(char **stringp, const char *delim) { } char *getpass(const char *prefix) { - static char passwd[TSDB_KEY_LEN] = {0}; - memset(passwd, 0, TSDB_KEY_LEN); + static char passwd[TSDB_PASSWORD_LEN] = {0}; + memset(passwd, 0, TSDB_PASSWORD_LEN); //printf("%s", prefix); int32_t index = 0; char ch; - while (index < TSDB_KEY_LEN) { + while (index < TSDB_PASSWORD_LEN) { ch = getch(); if (ch == '\n' || ch == '\r') { break; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index ca817c4c1e809bd3a74f2385167403de584f9da8..f892b4d8c0c95c0377274870110f16613012ef3d 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -714,7 +714,7 @@ static void taosGetSystemLocale() { // get and set default locale //printf("locale not configured, set to system default:%s", tsLocale); } - /* if user does not specify the charset, extract it from locale */ + // if user does not specify the charset, extract it from locale char *str = strrchr(tsLocale, sep); if (str != NULL) { str++; @@ -1118,13 +1118,13 @@ char *taosGetCmdlineByPID(int pid) { SysNameInfo taosGetSysNameInfo() { SysNameInfo info = {0}; - struct utsname buf; - if (!uname(&buf)) { - info.sysname = buf.sysname; - info.sysname == buf.nodename; - info.sysname = buf.release; - info.sysname = buf.version; - info.sysname = buf.machine; + struct utsname uts; + if (!uname(&uts)) { + info.sysname = strdup(uts.sysname); + info.nodename = strdup(uts.nodename); + info.release = strdup(uts.release); + info.version = strdup(uts.version); + info.machine = strdup(uts.machine); } return info; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8a6f389366d2393c4b38dd499755f9484f694b25..726247d4506c65d8a8f1597a61e8ef0e5c9002c2 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -282,7 +282,7 @@ static void taosReadConfigOption(const char *option, char *value, char *value2, } } -void taosInitConfigOption(SGlobalCfg cfg) { +void taosAddConfigOption(SGlobalCfg cfg) { tsGlobalConfig[tsGlobalConfigNum++] = cfg; } @@ -335,7 +335,7 @@ void taosReadGlobalLogCfg() { fclose(fp); } -int32_t taosReadGlobalCfg() { +int32_t taosReadCfgFromFile() { char * line, *option, *value, *value2, *value3; int olen, vlen, vlen2, vlen3; char fileName[PATH_MAX] = {0}; @@ -396,7 +396,7 @@ int32_t taosReadGlobalCfg() { return 0; } -void taosPrintGlobalCfg() { +void taosPrintCfg() { uInfo(" taos config & system info:"); uInfo("=================================="); @@ -443,7 +443,6 @@ void taosPrintGlobalCfg() { } taosPrintOsInfo(); - // taosPrintDataDirCfg(); uInfo("=================================="); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ceb344ffe8be2058f5909103828129697c7d68a0..42fde042e7a56e94b9afb7c6e57015a698614c13 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -127,6 +127,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_TAG_NAMES, "duplicated tag names" TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") @@ -497,6 +498,15 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, "tfs file already exis TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") +// catalog +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog interval error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INVALID_INPUT, "invalid catalog input parameters") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_NOT_READY, "catalog is not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_SYS_ERROR, "catalog system error") + + + #ifdef TAOS_ERROR_C }; #endif @@ -545,4 +555,4 @@ const char* tstrerror(int32_t err) { return ""; } -const char* terrstr() { return tstrerror(terrno); } \ No newline at end of file +const char* terrstr() { return tstrerror(terrno); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e6cc3a53afc6c037d33b4d8d3b71125af7fa3577..2d7179c0d319c8abad0a58a89e4456cb3fce33b1 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -99,6 +99,8 @@ int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; +int32_t ctgDebugFlag = 131; + int64_t dbgEmptyW = 0; int64_t dbgWN = 0; @@ -112,7 +114,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); -extern void taosPrintGlobalCfg(); +extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { @@ -222,7 +224,7 @@ static void *taosThreadToOpenNewFile(void *param) { uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintGlobalCfg(); + taosPrintCfg(); taosKeepOldLog(keepName); return NULL; diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c index 5606ab248d71b62f9b6041b76069a5292cd33223..d9356b7e4049cfdf9bc77ff1606e24993758bd6c 100644 --- a/source/util/src/tnote.c +++ b/source/util/src/tnote.c @@ -49,15 +49,6 @@ int32_t taosInitNotes() { taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); } - if (tsHttpEnableRecordSql) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/httpsql", tsLogDir); - taosInitNote(tsNumOfLogLines, 1, &tsHttpNote, name); - } - - if (tscEmbedded == 1) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/taosinfo", tsLogDir); - taosInitNote(tsNumOfLogLines, 1, &tsInfoNote, name); - } #endif return 0; } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 56186d9b24d5c6763f0c36714a030bc1abf71c1a..1fdc2257d787124d2bff5c9f23bc740bce922314 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -20,12 +20,10 @@ #include "tutil.h" #include "taoserror.h" -extern int8_t tscEmbedded; - -#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} +#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} #define tmrDebug(...) { if (tmrDebugFlag & DEBUG_DEBUG) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} #define tmrTrace(...) { if (tmrDebugFlag & DEBUG_TRACE) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index e8a5cdd3011ace34a18c89fd5706150f37413c5c..58a2e57f7c527cf8e75e8e0d3acf5beab0c4e6d7 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -417,16 +417,3 @@ void taosIp2String(uint32_t ip, char *str) { void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); } - -int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { - *port = 0; - strcpy(fqdn, ep); - - char *temp = strchr(fqdn, ':'); - if (temp) { - *temp = 0; - *port = atoi(temp + 1); - } - - return 0; -} \ No newline at end of file diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 7338cccfeee184e7f6834e41064e0c71fe5145a0..71efe33011080ee03d3072f68a50047bfebe231f 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -26,7 +26,7 @@ typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg typedef struct { int32_t vgId; char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; // size must same with SVnodeObj.db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] FCqWrite cqWrite; } SCqCfg; @@ -37,7 +37,7 @@ typedef struct { int32_t master; int32_t num; // number of continuous streams char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; struct SCqObj *pHead; diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index ec44a85d5b29c0471db64b0362126804ae73adec..974103052eb190b8f63b15b94482cb9ba4dda455 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17515,7 +17515,7 @@ fun:gaih_inet.constprop.0 fun:getaddrinfo fun:taosGetFqdn - fun:taosCheckGlobalCfg + fun:taosCheckAndPrintCfg fun:taos_init_imp } { @@ -17740,7 +17740,7 @@ fun:gaih_inet.constprop.7 fun:getaddrinfo fun:taosGetFqdn - fun:taosCheckGlobalCfg + fun:taosCheckAndPrintCfg fun:taos_init_imp } {