From fa0d64f14b427a61daf62c40736369cae4320ff3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 22 Nov 2021 10:03:41 +0800 Subject: [PATCH] add daemon lib --- include/common/taosmsg.h | 16 +- include/dnode/mgmt/dnode.h | 26 +- include/dnode/mnode/mnode.h | 21 +- include/dnode/vnode/vnode.h | 8 +- include/libs/transport/trpc.h | 10 +- include/util/taoserror.h | 41 +- source/dnode/mgmt/daemon/src/daemon.c | 171 ++- .../impl/inc/{dnodeVnodes.h => dndDnode.h} | 25 +- .../mgmt/impl/inc/{dnodeInt.h => dndInt.h} | 96 +- .../impl/inc/{dnodeDnode.h => dndMnode.h} | 25 +- .../inc/{dnodeTransport.h => dndTransport.h} | 16 +- .../impl/inc/{dnodeMnode.h => dndVnodes.h} | 24 +- .../impl/src/{dnodeDnode.c => dndDnode.c} | 430 ++++--- .../mgmt/impl/src/{dnodeInt.c => dndInt.c} | 145 +-- source/dnode/mgmt/impl/src/dndMnode.c | 816 ++++++++++++ source/dnode/mgmt/impl/src/dndTransport.c | 328 +++++ source/dnode/mgmt/impl/src/dndVnodes.c | 1119 +++++++++++++++++ source/dnode/mgmt/impl/src/dnodeMnode.c | 608 --------- source/dnode/mgmt/impl/src/dnodeTransport.c | 395 ------ source/dnode/mgmt/impl/src/dnodeVnodes.c | 1029 --------------- source/dnode/mnode/impl/inc/mnodeDef.h | 4 +- source/dnode/mnode/impl/src/mnode.c | 8 +- source/libs/transport/src/rpcMain.c | 24 +- source/util/src/terror.c | 25 +- 24 files changed, 2911 insertions(+), 2499 deletions(-) rename source/dnode/mgmt/impl/inc/{dnodeVnodes.h => dndDnode.h} (53%) rename source/dnode/mgmt/impl/inc/{dnodeInt.h => dndInt.h} (54%) rename source/dnode/mgmt/impl/inc/{dnodeDnode.h => dndMnode.h} (53%) rename source/dnode/mgmt/impl/inc/{dnodeTransport.h => dndTransport.h} (68%) rename source/dnode/mgmt/impl/inc/{dnodeMnode.h => dndVnodes.h} (52%) rename source/dnode/mgmt/impl/src/{dnodeDnode.c => dndDnode.c} (51%) rename source/dnode/mgmt/impl/src/{dnodeInt.c => dndInt.c} (56%) create mode 100644 source/dnode/mgmt/impl/src/dndMnode.c create mode 100644 source/dnode/mgmt/impl/src/dndTransport.c create mode 100644 source/dnode/mgmt/impl/src/dndVnodes.c delete mode 100644 source/dnode/mgmt/impl/src/dnodeMnode.c delete mode 100644 source/dnode/mgmt/impl/src/dnodeTransport.c delete mode 100644 source/dnode/mgmt/impl/src/dnodeVnodes.c diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index cf5a3c8e05..e70a8539c3 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -672,16 +672,16 @@ typedef struct { } SDnodeCfg; typedef struct { - int32_t dnodeId; + int32_t id; int8_t isMnode; int8_t reserved; - uint16_t dnodePort; - char dnodeFqdn[TSDB_FQDN_LEN]; + uint16_t port; + char fqdn[TSDB_FQDN_LEN]; } SDnodeEp; typedef struct { - int32_t dnodeNum; - SDnodeEp dnodeEps[]; + int32_t num; + SDnodeEp eps[]; } SDnodeEps; typedef struct { @@ -820,9 +820,9 @@ typedef struct { } SCreateDnodeMsg, SDropDnodeMsg; typedef struct { - int32_t dnodeId; - int8_t replica; - int8_t reserved[3]; + int32_t dnodeId; + int8_t replica; + int8_t reserved[3]; SReplica replicas[TSDB_MAX_REPLICA]; } SCreateMnodeMsg, SAlterMnodeMsg, SDropMnodeMsg; diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 5002ee37b0..a1a94bb10b 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -16,6 +16,8 @@ #ifndef _TD_DNODE_H_ #define _TD_DNODE_H_ +#include "tdef.h" + #ifdef __cplusplus extern "C" { #endif @@ -23,6 +25,24 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SDnode SDnode; +typedef struct { + int32_t sver; + int32_t numOfCores; + float numOfThreadsPerCore; + float ratioOfQueryCores; + int32_t maxShellConns; + int32_t shellActivityTimer; + int32_t statusInterval; + uint16_t serverPort; + char dataDir[PATH_MAX]; + char localEp[TSDB_EP_LEN]; + char localFqdn[TSDB_FQDN_LEN]; + char firstEp[TSDB_EP_LEN]; + char timezone[TSDB_TIMEZONE_LEN]; + char locale[TSDB_LOCALE_LEN]; + char charset[TSDB_LOCALE_LEN]; +} SDnodeOpt; + /* ------------------------ SDnode ------------------------ */ /** * @brief Initialize and start the dnode. @@ -30,14 +50,14 @@ typedef struct SDnode SDnode; * @param cfgPath Config file path. * @return SDnode* The dnode object. */ -SDnode *dnodeInit(const char *cfgPath); +SDnode *dndInit(SDnodeOpt *pOptions); /** * @brief Stop and cleanup dnode. * - * @param pDnode The dnode object to close. + * @param pDnd The dnode object to close. */ -void dnodeCleanup(SDnode *pDnode); +void dndCleanup(SDnode *pDnd); #ifdef __cplusplus } diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 0071296bc1..98aefc6db3 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -24,10 +24,10 @@ extern "C" { typedef struct SDnode SDnode; typedef struct SMnode SMnode; typedef struct SMnodeMsg SMnodeMsg; -typedef void (*SendMsgToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef void (*SendMsgToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectMsgFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg, bool forShell); -typedef int32_t (*PutMsgToMnodeQFp)(SDnode *pDnode, SMnodeMsg *pMsg); +typedef void (*SendMsgToDnodeFp)(SDnode *pDnd, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); +typedef void (*SendMsgToMnodeFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg); +typedef void (*SendRedirectMsgFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg, bool forShell); +typedef int32_t (*PutMsgToMnodeQFp)(SDnode *pDnd, SMnodeMsg *pMsg); typedef struct SMnodeLoad { int64_t numOfDnode; @@ -48,7 +48,7 @@ typedef struct { int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; - struct SServer *pServer; + struct SDnode *pDnode; PutMsgToMnodeQFp putMsgToApplyMsgFp; SendMsgToDnodeFp sendMsgToDnodeFp; SendMsgToMnodeFp sendMsgToMnodeFp; @@ -122,10 +122,17 @@ SMnodeMsg *mnodeInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg); /** * @brief Cleanup mnode msg * - * @param pMnode The mnode object * @param pMsg The request msg */ -void mnodeCleanupMsg(SMnode *pMnode, SMnodeMsg *pMsg); +void mnodeCleanupMsg(SMnodeMsg *pMsg); + +/** + * @brief Cleanup mnode msg + * + * @param pMsg The request msg + * @param code The error code + */ +void mnodeSendRsp(SMnodeMsg *pMsg, int32_t code); /** * @brief Process the read request diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 36f6a3b6fb..586cb49d0f 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -185,10 +185,10 @@ typedef struct { } SVnodeMsg; typedef struct SDnode SDnode; -typedef void (*SendMsgToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef void (*SendMsgToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectMsgFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg, bool forShell); -typedef int32_t (*PutMsgToVnodeQFp)(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg); +typedef void (*SendMsgToDnodeFp)(SDnode *pDnd, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); +typedef void (*SendMsgToMnodeFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg); +typedef void (*SendRedirectMsgFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg, bool forShell); +typedef int32_t (*PutMsgToVnodeQFp)(SDnode *pDnd, int32_t vgId, SVnodeMsg *pMsg); typedef struct { PutMsgToVnodeQFp putMsgToApplyQueueFp; diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 5460ae5401..2a8a7aad4b 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -51,7 +51,7 @@ typedef struct SRpcMsg { } SRpcMsg; typedef struct SRpcInit { - uint16_t localPort; // local port + uint16_t localPort; // local port char *label; // for debug purpose int numOfThreads; // number of threads to handle connections int sessions; // number of sessions allowed @@ -66,10 +66,12 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(SRpcMsg *, SEpSet *); + void (*cfp)(void *parent, SRpcMsg *, SEpSet *); - // call back to retrieve the client auth info, for server app only - int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + // call back to retrieve the client auth info, for server app only + int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + + void *parent; } SRpcInit; int32_t rpcInit(); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 022c3e9096..304fb56a6a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -68,12 +68,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0106) #define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0107) #define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0108) -#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0109) -#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x010A) -#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x010B) -#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x010C) -#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x010D) -#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010E) +#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) +#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) +#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) +#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) +#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0113) +#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0114) +#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0115) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid Operation") @@ -223,20 +224,20 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) //"Topic already exists) // dnode -#define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) //"Message not processed") -#define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) //"Dnode out of memory") -#define TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE TAOS_DEF_ERROR_CODE(0, 0x0402) //"Mnode Id not match Dnode") -#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) //"Mnode already deployed") -#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0404) //"Mnode not deployed") -#define TSDB_CODE_DND_READ_MNODE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0405) //"Read mnode.json error") -#define TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0406) //"Write mnode.json error") -#define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0407) //"No permission for disk files in dnode") -#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0408) //"Invalid message length") -#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0409) //"Action in progress") -#define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x040A) //"Too many vnode directories") -#define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x040B) //"Dnode is exiting" -#define TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x040C) //"Parse vnodes.json error") -#define TSDB_CODE_DND_PARSE_DNODE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x040D) //"Parse dnodes.json error") +#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) +#define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0401) +#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0402) +#define TSDB_CODE_DND_DNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0410) +#define TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0411) +#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0420) +#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0421) +#define TSDB_CODE_DND_MNODE_ID_INVALID TAOS_DEF_ERROR_CODE(0, 0x0422) +#define TSDB_CODE_DND_MNODE_ID_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0423) +#define TSDB_CODE_DND_MNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0424) +#define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0425) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0430) +#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0431) +#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0432) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 720d1589c2..326dfae4af 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -16,38 +16,173 @@ #define _DEFAULT_SOURCE #include "dnode.h" #include "os.h" +#include "tglobal.h" #include "ulog.h" -static bool stop = false; +static struct { + bool stop; + bool dumpConfig; + bool generateGrant; + bool printAuth; + bool printVersion; + char configDir[PATH_MAX]; +} global = {0}; -static void sigintHandler(int32_t signum, void *info, void *ctx) { stop = true; } +void dmnSigintHandle(int signum, void *info, void *ctx) { global.stop = true; } -static void setSignalHandler() { - taosSetSignal(SIGTERM, sigintHandler); - taosSetSignal(SIGHUP, sigintHandler); - taosSetSignal(SIGINT, sigintHandler); - taosSetSignal(SIGABRT, sigintHandler); - taosSetSignal(SIGBREAK, sigintHandler); +void dmnSetSignalHandle() { + taosSetSignal(SIGTERM, dmnSigintHandle); + taosSetSignal(SIGHUP, dmnSigintHandle); + taosSetSignal(SIGINT, dmnSigintHandle); + taosSetSignal(SIGABRT, dmnSigintHandle); + taosSetSignal(SIGBREAK, dmnSigintHandle); } -int main(int argc, char const *argv[]) { - const char *path = "/etc/taos"; +int dmnParseOpts(int argc, char const *argv[]) { + tstrncpy(global.configDir, "/etc/taos", PATH_MAX); - SDnode *pDnode = dnodeInit(path); - if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - exit(EXIT_FAILURE); + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(global.configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default:%s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-C") == 0) { + global.dumpConfig = true; + } else if (strcmp(argv[i], "-k") == 0) { + global.generateGrant = true; + } else if (strcmp(argv[i], "-A") == 0) { + global.printAuth = true; + } else if (strcmp(argv[i], "-V") == 0) { + global.printVersion = true; + } else { + } } - uInfo("Started TDengine service successfully."); + return 0; +} + +void dmnGenerateGrant() { grantParseParameter(); } + +void dmnPrintVersion() { +#ifdef TD_ENTERPRISE + char *versionStr = "enterprise"; +#else + char *versionStr = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", versionStr, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("gitinfoI: %s\n", gitinfoOfInternal); + printf("builuInfo: %s\n", buildinfo); +} + +int dmnReadConfig(const char *path) { + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + taosInitGlobalCfg(); + taosReadGlobalLogCfg(); + + if (taosMkDir(tsLogDir) != 0) { + printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); + return -1; + } + + char temp[PATH_MAX]; + snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir); + if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + printf("failed to init log file\n"); + return -1; + } + + if (taosInitNotes() != 0) { + printf("failed to init log file\n"); + return -1; + } + + if (taosReadGlobalCfg() != 0) { + uError("failed to read global config"); + return -1; + } + + if (taosCheckGlobalCfg() != 0) { + uError("failed to check global config"); + return -1; + } + + taosSetCoreDump(tsEnableCoreFile); + return 0; +} - setSignalHandler(); - while (!stop) { +void dmnDumpConfig() { taosDumpGlobalCfg(); } + +void dmnWaitSignal() { + dmnSetSignalHandle(); + while (!global.stop) { taosMsleep(100); } +} + +void dmnInitOption(SDnodeOpt *pOpt) { + pOpt->sver = tsVersion; + pOpt->numOfCores = tsNumOfCores; + pOpt->statusInterval = tsStatusInterval; + pOpt->serverPort = tsServerPort; + tstrncpy(pOpt->localEp, tsLocalEp, TSDB_EP_LEN); + tstrncpy(pOpt->localFqdn, tsLocalEp, TSDB_FQDN_LEN); + tstrncpy(pOpt->timezone, tsLocalEp, TSDB_TIMEZONE_LEN); + tstrncpy(pOpt->locale, tsLocalEp, TSDB_LOCALE_LEN); + tstrncpy(pOpt->charset, tsLocalEp, TSDB_LOCALE_LEN); +} + +int dmnRunDnode() { + SDnodeOpt opt = {0}; + dmnInitOption(&opt); + SDnode *pDnd = dndInit(&opt); + if (pDnd == NULL) { + uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); + return -1; + } + + uInfo("Started TDengine service successfully."); + dmnWaitSignal(); uInfo("TDengine is shut down!"); - dnodeCleanup(pDnode); + dndCleanup(pDnd); + taosCloseLog(); return 0; } + +int main(int argc, char const *argv[]) { + if (dmnParseOpts(argc, argv) != 0) { + return -1; + } + + if (global.generateGrant) { + dmnGenerateGrant(); + return 0; + } + + if (global.printVersion) { + dmnPrintVersion(); + return 0; + } + + if (dmnReadConfig(global.configDir) != 0) { + return -1; + } + + if (global.dumpConfig) { + dmnDumpConfig(); + return 0; + } + + return dmnRunDnode(); +} diff --git a/source/dnode/mgmt/impl/inc/dnodeVnodes.h b/source/dnode/mgmt/impl/inc/dndDnode.h similarity index 53% rename from source/dnode/mgmt/impl/inc/dnodeVnodes.h rename to source/dnode/mgmt/impl/inc/dndDnode.h index 31eae049ab..ef16b1c8f0 100644 --- a/source/dnode/mgmt/impl/inc/dnodeVnodes.h +++ b/source/dnode/mgmt/impl/inc/dndDnode.h @@ -13,26 +13,27 @@ * along with this program. If not, see . */ -#ifndef _TD_DNODE_VNODES_H_ -#define _TD_DNODE_VNODES_H_ +#ifndef _TD_DND_DNODE_H_ +#define _TD_DND_DNODE_H_ #ifdef __cplusplus extern "C" { #endif -#include "dnodeInt.h" +#include "dndInt.h" -int32_t dnodeInitVnodes(); -void dnodeCleanupVnodes(); -void dnodeGetVnodeLoads(SVnodeLoads *pVloads); +int32_t dndInitDnode(SDnode *pDnd); +void dndCleanupDnode(SDnode *pDnd); +void dndProcessDnodeReq(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessDnodeRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet); -void dnodeProcessVnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet); -void dnodeProcessVnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet); -void dnodeProcessVnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet); -void dnodeProcessVnodeQueryMsg(SRpcMsg *pMsg, SEpSet *pEpSet); -void dnodeProcessVnodeFetchMsg(SRpcMsg *pMsg, SEpSet *pEpSet); +int32_t dndGetDnodeId(SDnode *pDnd); +int64_t dndGetClusterId(SDnode *pDnd); +void dndGetDnodeEp(SDnode *pDnd, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort); +void dndGetMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet); +void dndSendRedirectMsg(SDnode *pDnd, SRpcMsg *pMsg, bool forShell); #ifdef __cplusplus } #endif -#endif /*_TD_DNODE_VNODES_H_*/ \ No newline at end of file +#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dnodeInt.h b/source/dnode/mgmt/impl/inc/dndInt.h similarity index 54% rename from source/dnode/mgmt/impl/inc/dnodeInt.h rename to source/dnode/mgmt/impl/inc/dndInt.h index 8944755268..966781426b 100644 --- a/source/dnode/mgmt/impl/inc/dnodeInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -13,20 +13,27 @@ * along with this program. If not, see . */ -#ifndef _TD_DNODE_INT_H_ -#define _TD_DNODE_INT_H_ +#ifndef _TD_DND_INT_H_ +#define _TD_DND_INT_H_ #ifdef __cplusplus extern "C" { #endif + +#include "cJSON.h" #include "os.h" #include "taosmsg.h" -#include "tglobal.h" #include "thash.h" +#include "tlockfree.h" #include "tlog.h" +#include "tqueue.h" #include "trpc.h" #include "tthread.h" #include "ttime.h" +#include "tworker.h" +#include "mnode.h" +#include "vnode.h" +#include "dnode.h" extern int32_t dDebugFlag; @@ -37,8 +44,8 @@ extern int32_t dDebugFlag; #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("SRV ", dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("SRV ", dDebugFlag, __VA_ARGS__); }} -typedef enum { DN_STAT_INIT, DN_STAT_RUNNING, DN_STAT_STOPPED } EStat; -typedef void (*MsgFp)(SRpcMsg *pMsg, SEpSet *pEpSet); +typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; +typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); typedef struct { char *dnode; @@ -48,50 +55,75 @@ typedef struct { typedef struct { int32_t dnodeId; + uint32_t rebootTime; + int32_t dropped; int64_t clusterId; - SDnodeEps *dnodeEps; - SHashObj *dnodeHash; - SEpSet mnodeEpSetForShell; - SEpSet mnodeEpSetForPeer; + SEpSet shellEpSet; + SEpSet peerEpSet; char *file; - uint32_t rebootTime; - int8_t dropped; - int8_t threadStop; + SHashObj *dnodeHash; + SDnodeEps *dnodeEps; pthread_t *threadId; pthread_mutex_t mutex; -} SDnodeDnode; +} SDnodeMgmt; typedef struct { -} SDnodeMnode; + int32_t refCount; + int8_t deployed; + int8_t dropped; + SWorkerPool mgmtPool; + SWorkerPool readPool; + SWorkerPool writePool; + SWorkerPool syncPool; + taos_queue pReadQ; + taos_queue pWriteQ; + taos_queue pApplyQ; + taos_queue pSyncQ; + taos_queue pMgmtQ; + char *file; + SMnode *pMnode; + SRWLatch latch; +} SMnodeMgmt; typedef struct { -} SDnodeVnodes; + SHashObj *hash; + SWorkerPool mgmtPool; + SWorkerPool queryPool; + SWorkerPool fetchPool; + SMWorkerPool syncPool; + SMWorkerPool writePool; + taos_queue pMgmtQ; + int32_t openVnodes; + int32_t totalVnodes; + SRWLatch latch; +} SVnodesMgmt; typedef struct { - void *peerRpc; - void *shellRpc; - void *clientRpc; -} SDnodeTrans; + void *serverRpc; + void *clientRpc; + DndMsgFp msgFp[TSDB_MSG_TYPE_MAX]; +} STransMgmt; typedef struct SDnode { - EStat stat; - SDnodeDir dir; - SDnodeDnode dnode; - SDnodeVnodes vnodes; - SDnodeMnode mnode; - SDnodeTrans trans; - SStartupMsg startup; + EStat stat; + SDnodeOpt opt; + SDnodeDir dir; + SDnodeMgmt d; + SMnodeMgmt m; + SVnodesMgmt vmgmt; + STransMgmt t; + SStartupMsg startup; } SDnode; -EStat dnodeGetStat(SDnode *pDnode); -void dnodeSetStat(SDnode *pDnode, EStat stat); -char *dnodeStatStr(EStat stat); +EStat dndGetStat(SDnode *pDnode); +void dndSetStat(SDnode *pDnode, EStat stat); +char *dndStatStr(EStat stat); -void dnodeReportStartup(SDnode *pDnode, char *name, char *desc); -void dnodeGetStartup(SDnode *pDnode, SStartupMsg *pStartup); +void dndReportStartup(SDnode *pDnode, char *name, char *desc); +void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup); #ifdef __cplusplus } #endif -#endif /*_TD_DNODE_INT_H_*/ \ No newline at end of file +#endif /*_TD_DND_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dnodeDnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h similarity index 53% rename from source/dnode/mgmt/impl/inc/dnodeDnode.h rename to source/dnode/mgmt/impl/inc/dndMnode.h index 87dc0fdb9b..67c51e51a8 100644 --- a/source/dnode/mgmt/impl/inc/dnodeDnode.h +++ b/source/dnode/mgmt/impl/inc/dndMnode.h @@ -13,27 +13,24 @@ * along with this program. If not, see . */ -#ifndef _TD_DNODE_DNODE_H_ -#define _TD_DNODE_DNODE_H_ +#ifndef _TD_DND_MNODE_H_ +#define _TD_DND_MNODE_H_ #ifdef __cplusplus extern "C" { #endif -#include "dnodeInt.h" +#include "dndInt.h" -int32_t dnodeInitDnode(SDnode *pDnode); -void dnodeCleanupDnode(SDnode *pDnode); -void dnodeProcessDnodeMsg(SRpcMsg *pMsg, SEpSet *pEpSet); - -int32_t dnodeGetDnodeId(); -int64_t dnodeGetClusterId(); -void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); -void dnodeGetMnodeEpSetForPeer(SEpSet *epSet); -void dnodeGetMnodeEpSetForShell(SEpSet *epSet); -void dnodeSendRedirectMsg(SDnode *pDnode, SRpcMsg *rpcMsg, bool forShell); +int32_t dndInitMnode(SDnode *pDnode); +void dndCleanupMnode(SDnode *pDnode); +int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); +void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); #ifdef __cplusplus } #endif -#endif /*_TD_DNODE_DNODE_H_*/ \ No newline at end of file +#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dnodeTransport.h b/source/dnode/mgmt/impl/inc/dndTransport.h similarity index 68% rename from source/dnode/mgmt/impl/inc/dnodeTransport.h rename to source/dnode/mgmt/impl/inc/dndTransport.h index 7d3f4be1ff..312da69fa2 100644 --- a/source/dnode/mgmt/impl/inc/dnodeTransport.h +++ b/source/dnode/mgmt/impl/inc/dndTransport.h @@ -13,21 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_DNODE_TRANSPORT_H_ -#define _TD_DNODE_TRANSPORT_H_ +#ifndef _TD_DND_TRANSPORT_H_ +#define _TD_DND_TRANSPORT_H_ #ifdef __cplusplus extern "C" { #endif -#include "dnodeInt.h" +#include "dndInt.h" -int32_t dnodeInitTrans(); -void dnodeCleanupTrans(); -void dnodeSendMsgToMnode(SDnode *pDnode, SRpcMsg *rpcMsg); -void dnodeSendMsgToDnode(SDnode *pDnode, SEpSet *epSet, SRpcMsg *rpcMsg); +int32_t dndInitTrans(SDnode *pDnode); +void dndCleanupTrans(SDnode *pDnode); +void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pRpcMsg); +void dndSendMsgToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pRpcMsg); #ifdef __cplusplus } #endif -#endif /*_TD_DNODE_TRANSPORT_H_*/ +#endif /*_TD_DND_TRANSPORT_H_*/ diff --git a/source/dnode/mgmt/impl/inc/dnodeMnode.h b/source/dnode/mgmt/impl/inc/dndVnodes.h similarity index 52% rename from source/dnode/mgmt/impl/inc/dnodeMnode.h rename to source/dnode/mgmt/impl/inc/dndVnodes.h index 9f52e586fe..35f99ee73b 100644 --- a/source/dnode/mgmt/impl/inc/dnodeMnode.h +++ b/source/dnode/mgmt/impl/inc/dndVnodes.h @@ -13,25 +13,25 @@ * along with this program. If not, see . */ -#ifndef _TD_DNODE_MNODE_H_ -#define _TD_DNODE_MNODE_H_ +#ifndef _TD_DND_VNODES_H_ +#define _TD_DND_VNODES_H_ #ifdef __cplusplus extern "C" { #endif -#include "dnodeInt.h" +#include "dndInt.h" -int32_t dnodeInitMnode(); -void dnodeCleanupMnode(); -int32_t dnodeGetUserAuthFromMnode(char *user, char *spi, char *encrypt, char *secret, char *ckey); - -void dnodeProcessMnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet); -void dnodeProcessMnodeReadMsg(SRpcMsg *pMsg, SEpSet *pEpSet); -void dnodeProcessMnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet); -void dnodeProcessMnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet); +int32_t dndInitVnodes(SDnode *pDnode); +void dndCleanupVnodes(SDnode *pDnode); +void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pVloads); +void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); #ifdef __cplusplus } #endif -#endif /*_TD_DNODE_MNODE_H_*/ \ No newline at end of file +#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dnodeDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c similarity index 51% rename from source/dnode/mgmt/impl/src/dnodeDnode.c rename to source/dnode/mgmt/impl/src/dndDnode.c index ec60116ce6..378d76e046 100644 --- a/source/dnode/mgmt/impl/src/dnodeDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -14,69 +14,76 @@ */ #define _DEFAULT_SOURCE -#include "dnodeDnode.h" -#include "dnodeTransport.h" -#include "dnodeVnodes.h" -#include "cJSON.h" - -int32_t dnodeGetDnodeId() { - int32_t dnodeId = 0; - pthread_mutex_lock(&pDnode->mutex); - dnodeId = pDnode->dnodeId; - pthread_mutex_unlock(&pDnode->mutex); +#include "dndDnode.h" +#include "dndTransport.h" +#include "dndVnodes.h" + +static inline void dndLockDnode(SDnode *pDnd) { pthread_mutex_lock(&pDnd->d.mutex); } + +static inline void dndUnLockDnode(SDnode *pDnd) { pthread_mutex_unlock(&pDnd->d.mutex); } + +int32_t dndGetDnodeId(SDnode *pDnd) { + dndLockDnode(pDnd); + int32_t dnodeId = pDnd->d.dnodeId; + dndUnLockDnode(pDnd); return dnodeId; } -int64_t dnodeGetClusterId() { - int64_t clusterId = 0; - pthread_mutex_lock(&pDnode->mutex); - clusterId = pDnode->clusterId; - pthread_mutex_unlock(&pDnode->mutex); +int64_t dndGetClusterId(SDnode *pDnd) { + dndLockDnode(pDnd); + int64_t clusterId = pDnd->d.clusterId; + dndUnLockDnode(pDnd); return clusterId; } -void dnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port) { - pthread_mutex_lock(&pDnode->mutex); +void dndGetDnodeEp(SDnode *pDnd, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { + dndLockDnode(pDnd); - SDnodeEp *pEp = taosHashGet(pDnode->dnodeHash, &dnodeId, sizeof(int32_t)); - if (pEp != NULL) { - if (port) *port = pEp->dnodePort; - if (fqdn) tstrncpy(fqdn, pEp->dnodeFqdn, TSDB_FQDN_LEN); - if (ep) snprintf(ep, TSDB_EP_LEN, "%s:%u", pEp->dnodeFqdn, pEp->dnodePort); + SDnodeEp *pDnodeEp = taosHashGet(pDnd->d.dnodeHash, &dnodeId, sizeof(int32_t)); + if (pDnodeEp != NULL) { + if (pPort != NULL) { + *pPort = pDnodeEp->port; + } + if (pFqdn != NULL) { + tstrncpy(pFqdn, pDnodeEp->fqdn, TSDB_FQDN_LEN); + } + if (pEp != NULL) { + snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port); + } } - pthread_mutex_unlock(&pDnode->mutex); + dndUnLockDnode(pDnd); } -void dnodeGetMnodeEpSetForPeer(SEpSet *pEpSet) { - pthread_mutex_lock(&pDnode->mutex); - *pEpSet = pDnode->mnodeEpSetForPeer; - pthread_mutex_unlock(&pDnode->mutex); +void dndGetMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet) { + dndLockDnode(pDnd); + *pEpSet = pDnd->d.peerEpSet; + dndUnLockDnode(pDnd); } -void dnodeGetMnodeEpSetForShell(SEpSet *pEpSet) { - pthread_mutex_lock(&pDnode->mutex); - *pEpSet = pDnode->mnodeEpSetForShell; - pthread_mutex_unlock(&pDnode->mutex); +void dndGetShellEpSet(SDnode *pDnd, SEpSet *pEpSet) { + dndLockDnode(pDnd); + *pEpSet = pDnd->d.shellEpSet; + dndUnLockDnode(pDnd); } -void dnodeSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg, bool forShell) { +void dndSendRedirectMsg(SDnode *pDnd, SRpcMsg *pMsg, bool forShell) { int32_t msgType = pMsg->msgType; SEpSet epSet = {0}; if (forShell) { - dnodeGetMnodeEpSetForShell(&epSet); + dndGetShellEpSet(pDnd, &epSet); } else { - dnodeGetMnodeEpSetForPeer(&epSet); + dndGetMnodeEpSet(pDnd, &epSet); } dDebug("RPC %p, msg:%s is redirected, num:%d use:%d", pMsg->handle, taosMsg[msgType], epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]); - if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) { - if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) || - (epSet.port[i] == tsServerPort && forShell)) { + if (strcmp(epSet.fqdn[i], pDnd->opt.localFqdn) == 0) { + if ((epSet.port[i] == pDnd->opt.serverPort + TSDB_PORT_DNODEDNODE && !forShell) || + (epSet.port[i] == pDnd->opt.serverPort && forShell)) { epSet.inUse = (i + 1) % epSet.numOfEps; dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse); } @@ -88,39 +95,37 @@ void dnodeSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg, bool forShell) { rpcSendRedirectRsp(pMsg->handle, &epSet); } -static void dnodeUpdateMnodeEpSet(SDnodeDnode *pDnode, SEpSet *pEpSet) { - if (pEpSet == NULL || pEpSet->numOfEps <= 0) { - dError("mnode is changed, but content is invalid, discard it"); - return; - } else { - dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); - } +static void dndUpdateMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet) { + dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse); - pthread_mutex_lock(&pDnode->mutex); + dndLockDnode(pDnd); - pDnode->mnodeEpSetForPeer = *pEpSet; + pDnd->d.peerEpSet = *pEpSet; for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { pEpSet->port[i] -= TSDB_PORT_DNODEDNODE; dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]); } - pDnode->mnodeEpSetForShell = *pEpSet; + pDnd->d.shellEpSet = *pEpSet; - pthread_mutex_unlock(&pDnode->mutex); + dndUnLockDnode(pDnd); } -static void dnodePrintDnodes() { - dDebug("print dnode endpoint list, num:%d", pDnode->dnodeEps->dnodeNum); - for (int32_t i = 0; i < pDnode->dnodeEps->dnodeNum; i++) { - SDnodeEp *ep = &pDnode->dnodeEps->dnodeEps[i]; - dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort, ep->isMnode); +static void dndPrintDnodes(SDnode *pDnd) { + SDnodeMgmt *pDnode = &pDnd->d; + + dDebug("print dnode endpoint list, num:%d", pDnode->dnodeEps->num); + for (int32_t i = 0; i < pDnode->dnodeEps->num; i++) { + SDnodeEp *pEp = &pDnode->dnodeEps->eps[i]; + dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->fqdn, pEp->port, pEp->isMnode); } } -static void dnodeResetDnodes(SDnodeEps *pEps) { - assert(pEps != NULL); - int32_t size = sizeof(SDnodeEps) + pEps->dnodeNum * sizeof(SDnodeEp); +static void dndResetDnodes(SDnode *pDnd, SDnodeEps *pDnodeEps) { + SDnodeMgmt *pDnode = &pDnd->d; - if (pEps->dnodeNum > pDnode->dnodeEps->dnodeNum) { + int32_t size = sizeof(SDnodeEps) + pDnodeEps->num * sizeof(SDnodeEp); + + if (pDnodeEps->num > pDnode->dnodeEps->num) { SDnodeEps *tmp = calloc(1, size); if (tmp == NULL) return; @@ -128,49 +133,51 @@ static void dnodeResetDnodes(SDnodeEps *pEps) { pDnode->dnodeEps = tmp; } - if (pDnode->dnodeEps != pEps) { - memcpy(pDnode->dnodeEps, pEps, size); + if (pDnode->dnodeEps != pDnodeEps) { + memcpy(pDnode->dnodeEps, pDnodeEps, size); } - pDnode->mnodeEpSetForPeer.inUse = 0; - pDnode->mnodeEpSetForShell.inUse = 0; + pDnode->peerEpSet.inUse = 0; + pDnode->shellEpSet.inUse = 0; int32_t mIndex = 0; - for (int32_t i = 0; i < pDnode->dnodeEps->dnodeNum; i++) { - SDnodeEp *ep = &pDnode->dnodeEps->dnodeEps[i]; - if (!ep->isMnode) continue; + for (int32_t i = 0; i < pDnode->dnodeEps->num; i++) { + SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i]; + if (!pDnodeEp->isMnode) continue; if (mIndex >= TSDB_MAX_REPLICA) continue; - strcpy(pDnode->mnodeEpSetForShell.fqdn[mIndex], ep->dnodeFqdn); - strcpy(pDnode->mnodeEpSetForPeer.fqdn[mIndex], ep->dnodeFqdn); - pDnode->mnodeEpSetForShell.port[mIndex] = ep->dnodePort; - pDnode->mnodeEpSetForShell.port[mIndex] = ep->dnodePort + tsDnodeDnodePort; + strcpy(pDnode->shellEpSet.fqdn[mIndex], pDnodeEp->fqdn); + strcpy(pDnode->peerEpSet.fqdn[mIndex], pDnodeEp->fqdn); + pDnode->shellEpSet.port[mIndex] = pDnodeEp->port; + pDnode->shellEpSet.port[mIndex] = pDnodeEp->port + TSDB_PORT_DNODEDNODE; mIndex++; } - for (int32_t i = 0; i < pDnode->dnodeEps->dnodeNum; ++i) { - SDnodeEp *ep = &pDnode->dnodeEps->dnodeEps[i]; - taosHashPut(pDnode->dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp)); + for (int32_t i = 0; i < pDnode->dnodeEps->num; ++i) { + SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i]; + taosHashPut(pDnode->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); } - dnodePrintDnodes(); + dndPrintDnodes(pDnd); } -static bool dnodeIsEpChanged(int32_t dnodeId, char *epStr) { +static bool dndIsEpChanged(SDnode *pDnd, int32_t dnodeId) { bool changed = false; - pthread_mutex_lock(&pDnode->mutex); + dndLockDnode(pDnd); - SDnodeEp *pEp = taosHashGet(pDnode->dnodeHash, &dnodeId, sizeof(int32_t)); - if (pEp != NULL) { - char epSaved[TSDB_EP_LEN + 1]; - snprintf(epSaved, TSDB_EP_LEN, "%s:%u", pEp->dnodeFqdn, pEp->dnodePort); - changed = strcmp(epStr, epSaved) != 0; + SDnodeEp *pDnodeEp = taosHashGet(pDnd->d.dnodeHash, &dnodeId, sizeof(int32_t)); + if (pDnodeEp != NULL) { + char epstr[TSDB_EP_LEN + 1]; + snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port); + changed = strcmp(pDnd->opt.localEp, epstr) != 0; } - pthread_mutex_unlock(&pDnode->mutex); + dndUnLockDnode(pDnd); return changed; } -static int32_t dnodeReadDnodes() { +static int32_t dndReadDnodes(SDnode *pDnd) { + SDnodeMgmt *pDnode = &pDnd->d; + int32_t len = 0; int32_t maxLen = 30000; char *content = calloc(1, maxLen + 1); @@ -234,70 +241,72 @@ static int32_t dnodeReadDnodes() { dError("failed to calloc dnodeEpList since %s", strerror(errno)); goto PRASE_DNODE_OVER; } - pDnode->dnodeEps->dnodeNum = dnodeInfosSize; + pDnode->dnodeEps->num = dnodeInfosSize; for (int32_t i = 0; i < dnodeInfosSize; ++i) { cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i); if (dnodeInfo == NULL) break; - SDnodeEp *pEp = &pDnode->dnodeEps->dnodeEps[i]; + SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i]; cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId"); if (!dnodeId || dnodeId->type != cJSON_String) { dError("failed to read %s, dnodeId not found", pDnode->file); goto PRASE_DNODE_OVER; } - pEp->dnodeId = atoi(dnodeId->valuestring); + pDnodeEp->id = atoi(dnodeId->valuestring); cJSON *isMnode = cJSON_GetObjectItem(dnodeInfo, "isMnode"); if (!isMnode || isMnode->type != cJSON_String) { dError("failed to read %s, isMnode not found", pDnode->file); goto PRASE_DNODE_OVER; } - pEp->isMnode = atoi(isMnode->valuestring); + pDnodeEp->isMnode = atoi(isMnode->valuestring); cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { dError("failed to read %s, dnodeFqdn not found", pDnode->file); goto PRASE_DNODE_OVER; } - tstrncpy(pEp->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + tstrncpy(pDnodeEp->fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort"); if (!dnodePort || dnodePort->type != cJSON_String) { dError("failed to read %s, dnodePort not found", pDnode->file); goto PRASE_DNODE_OVER; } - pEp->dnodePort = atoi(dnodePort->valuestring); + pDnodeEp->port = atoi(dnodePort->valuestring); } dInfo("succcessed to read file %s", pDnode->file); - dnodePrintDnodes(); + dndPrintDnodes(pDnd); PRASE_DNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); if (fp != NULL) fclose(fp); - if (dnodeIsEpChanged(pDnode->dnodeId, tsLocalEp)) { - dError("localEp %s different with %s and need reconfigured", tsLocalEp, pDnode->file); + if (dndIsEpChanged(pDnd, pDnode->dnodeId)) { + dError("localEp %s different with %s and need reconfigured", pDnd->opt.localEp, pDnode->file); return -1; } if (pDnode->dnodeEps == NULL) { pDnode->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp)); - pDnode->dnodeEps->dnodeNum = 1; - pDnode->dnodeEps->dnodeEps[0].dnodePort = tsServerPort; - tstrncpy(pDnode->dnodeEps->dnodeEps[0].dnodeFqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pDnode->dnodeEps->num = 1; + pDnode->dnodeEps->eps[0].port = pDnd->opt.serverPort; + tstrncpy(pDnode->dnodeEps->eps[0].fqdn, pDnd->opt.localFqdn, TSDB_FQDN_LEN); } - dnodeResetDnodes(pDnode->dnodeEps); + dndResetDnodes(pDnd, pDnode->dnodeEps); terrno = 0; return 0; } -static int32_t dnodeWriteDnodes() { +static int32_t dndWriteDnodes(SDnode *pDnd) { + SDnodeMgmt *pDnode = &pDnd->d; + FILE *fp = fopen(pDnode->file, "w"); if (!fp) { dError("failed to write %s since %s", pDnode->file, strerror(errno)); @@ -313,13 +322,13 @@ static int32_t dnodeWriteDnodes() { len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pDnode->clusterId); len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\",\n", pDnode->dropped); len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n"); - for (int32_t i = 0; i < pDnode->dnodeEps->dnodeNum; ++i) { - SDnodeEp *ep = &pDnode->dnodeEps->dnodeEps[i]; - len += snprintf(content + len, maxLen - len, " \"dnodeId\": \"%d\",\n", ep->dnodeId); - len += snprintf(content + len, maxLen - len, " \"isMnode\": \"%d\",\n", ep->isMnode); - len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn); - len += snprintf(content + len, maxLen - len, " \"dnodePort\": \"%u\"\n", ep->dnodePort); - if (i < pDnode->dnodeEps->dnodeNum - 1) { + for (int32_t i = 0; i < pDnode->dnodeEps->num; ++i) { + SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i]; + len += snprintf(content + len, maxLen - len, " \"dnodeId\": \"%d\",\n", pDnodeEp->id); + len += snprintf(content + len, maxLen - len, " \"isMnode\": \"%d\",\n", pDnodeEp->isMnode); + len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", pDnodeEp->fqdn); + len += snprintf(content + len, maxLen - len, " \"dnodePort\": \"%u\"\n", pDnodeEp->port); + if (i < pDnode->dnodeEps->num - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { len += snprintf(content + len, maxLen - len, " }]\n"); @@ -337,107 +346,114 @@ static int32_t dnodeWriteDnodes() { return 0; } -static void dnodeSendStatusMsg() { - int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); - +static void dndSendStatusMsg(SDnode *pDnd) { + int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); SStatusMsg *pStatus = rpcMallocCont(contLen); if (pStatus == NULL) { dError("failed to malloc status message"); return; } - pStatus->sversion = htonl(tsVersion); - pStatus->dnodeId = htonl(dnodeGetDnodeId()); - pStatus->clusterId = htobe64(dnodeGetClusterId()); - pStatus->rebootTime = htonl(pDnode->rebootTime); - pStatus->numOfCores = htonl(tsNumOfCores); - tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); - - pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); + dndLockDnode(pDnd); + pStatus->sversion = htonl(pDnd->opt.sver); + pStatus->dnodeId = htonl(pDnd->d.dnodeId); + pStatus->clusterId = htobe64(pDnd->d.clusterId); + pStatus->rebootTime = htonl(pDnd->d.rebootTime); + pStatus->numOfCores = htonl(pDnd->opt.numOfCores); + tstrncpy(pStatus->dnodeEp, pDnd->opt.localEp, TSDB_EP_LEN); + pStatus->clusterCfg.statusInterval = htonl(pDnd->opt.statusInterval); + tstrncpy(pStatus->clusterCfg.timezone, pDnd->opt.timezone, TSDB_TIMEZONE_LEN); + tstrncpy(pStatus->clusterCfg.locale, pDnd->opt.locale, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.charset, pDnd->opt.charset, TSDB_LOCALE_LEN); pStatus->clusterCfg.checkTime = 0; - tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + dndUnLockDnode(pDnd); - dnodeGetVnodeLoads(&pStatus->vnodeLoads); + dndGetVnodeLoads(pDnd, &pStatus->vnodeLoads); contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad); SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS}; - dnodeSendMsgToMnode(NULL, &rpcMsg); + dndSendMsgToMnode(pDnd, &rpcMsg); } -static void dnodeUpdateCfg(SDnodeCfg *pCfg) { - if (pDnode->dnodeId == 0) return; - if (pDnode->dropped) return; +static void dndUpdateDnodeCfg(SDnode *pDnd, SDnodeCfg *pCfg) { + SDnodeMgmt *pDnode = &pDnd->d; + if (pDnode->dnodeId != 0 && pDnode->dropped != pCfg->dropped) return; - pthread_mutex_lock(&pDnode->mutex); + dndLockDnode(pDnd); pDnode->dnodeId = pCfg->dnodeId; pDnode->clusterId = pCfg->clusterId; pDnode->dropped = pCfg->dropped; - dInfo("dnodeId is set to %d, clusterId is set to %" PRId64, pCfg->dnodeId, pCfg->clusterId); + dInfo("set dnodeId:%d clusterId:%" PRId64 " dropped:%d", pCfg->dnodeId, pCfg->clusterId, pCfg->dropped); - dnodeWriteDnodes(); - pthread_mutex_unlock(&pDnode->mutex); + dndWriteDnodes(pDnd); + dndUnLockDnode(pDnd); } -static void dnodeUpdateDnodeEps(SDnodeEps *pEps) { - if (pEps == NULL || pEps->dnodeNum <= 0) return; +static void dndUpdateDnodeEps(SDnode *pDnd, SDnodeEps *pDnodeEps) { + if (pDnodeEps == NULL || pDnodeEps->num <= 0) return; - pthread_mutex_lock(&pDnode->mutex); + dndLockDnode(pDnd); - if (pEps->dnodeNum != pDnode->dnodeEps->dnodeNum) { - dnodeResetDnodes(pEps); - dnodeWriteDnodes(); + if (pDnodeEps->num != pDnd->d.dnodeEps->num) { + dndResetDnodes(pDnd, pDnodeEps); + dndWriteDnodes(pDnd); } else { - int32_t size = pEps->dnodeNum * sizeof(SDnodeEp) + sizeof(SDnodeEps); - if (memcmp(pDnode->dnodeEps, pEps, size) != 0) { - dnodeResetDnodes(pEps); - dnodeWriteDnodes(); + int32_t size = pDnodeEps->num * sizeof(SDnodeEp) + sizeof(SDnodeEps); + if (memcmp(pDnd->d.dnodeEps, pDnodeEps, size) != 0) { + dndResetDnodes(pDnd, pDnodeEps); + dndWriteDnodes(pDnd); } } - pthread_mutex_unlock(&pDnode->mutex); + dndUnLockDnode(pDnd); } -static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { +static void dndProcessStatusRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { + if (pEpSet && pEpSet->numOfEps > 0) { + dndUpdateMnodeEpSet(pDnd, pEpSet); + } + if (pMsg->code != TSDB_CODE_SUCCESS) return; SStatusRsp *pStatusRsp = pMsg->pCont; - - SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; + SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->clusterId = htobe64(pCfg->clusterId); - dnodeUpdateCfg(pCfg); + dndUpdateDnodeCfg(pDnd, pCfg); if (pCfg->dropped) return; - SDnodeEps *pEps = &pStatusRsp->dnodeEps; - pEps->dnodeNum = htonl(pEps->dnodeNum); - for (int32_t i = 0; i < pEps->dnodeNum; ++i) { - pEps->dnodeEps[i].dnodeId = htonl(pEps->dnodeEps[i].dnodeId); - pEps->dnodeEps[i].dnodePort = htons(pEps->dnodeEps[i].dnodePort); + SDnodeEps *pDnodeEps = &pStatusRsp->dnodeEps; + pDnodeEps->num = htonl(pDnodeEps->num); + for (int32_t i = 0; i < pDnodeEps->num; ++i) { + pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id); + pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port); } - dnodeUpdateDnodeEps(pEps); + dndUpdateDnodeEps(pDnd, pDnodeEps); } -static void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg) { +static void dndProcessAuthRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); } + +static void dndProcessGrantRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); } + +static void dndProcessConfigDnodeReq(SDnode *pDnd, SRpcMsg *pMsg) { SCfgDnodeMsg *pCfg = pMsg->pCont; - int32_t code = taosCfgDynamicOptions(pCfg->config); + int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code}; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); } -static void dnodeProcessStartupReq(SRpcMsg *pMsg) { - dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont); +static void dndProcessStartupReq(SDnode *pDnd, SRpcMsg *pMsg) { + dInfo("startup msg is received"); SStartupMsg *pStartup = rpcMallocCont(sizeof(SStartupMsg)); - dnodeGetStartup(NULL, pStartup); + dndGetStartup(pDnd, pStartup); dInfo("startup msg is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); @@ -447,108 +463,120 @@ static void dnodeProcessStartupReq(SRpcMsg *pMsg) { } static void *dnodeThreadRoutine(void *param) { - int32_t ms = tsStatusInterval * 1000; + SDnode *pDnd = param; + int32_t ms = pDnd->opt.statusInterval * 1000; - while (!pDnode->threadStop) { - if (dnodeGetStat() != DN_STAT_RUNNING) { + while (true) { + taosMsleep(ms); + if (dndGetStat(pDnd) != DND_STAT_RUNNING) { continue; - } else { - dnodeSendStatusMsg(); } - taosMsleep(ms); + + pthread_testcancel(); + dndSendStatusMsg(pDnd); } } -int32_t dnodeInitDnode(SDnode *pServer) { - SDnodeDnode *pDnode = &pServer->dnode; +int32_t dndInitDnode(SDnode *pDnd) { + SDnodeMgmt *pDnode = &pDnd->d; + + pDnode->dnodeId = 0; + pDnode->rebootTime = taosGetTimestampSec(); + pDnode->dropped = 0; + pDnode->clusterId = 0; char path[PATH_MAX]; - snprintf(path, PATH_MAX, "%s/dnode.json", pServer->dir.dnode); + snprintf(path, PATH_MAX, "%s/dnode.json", pDnd->dir.dnode); pDnode->file = strdup(path); if (pDnode->file == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pDnode->dnodeId = 0; - pDnode->clusterId = 0; - pDnode->dnodeEps = NULL; - - pDnode->rebootTime = taosGetTimestampSec(); - pDnode->dropped = 0; - pthread_mutex_init(&pDnode->mutex, NULL); - pDnode->threadStop = false; - pDnode->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (pDnode->dnodeHash == NULL) { dError("failed to init dnode hash"); - return TSDB_CODE_DND_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - pDnode->threadId = taosCreateThread(dnodeThreadRoutine, NULL); - if (pDnode->threadId == NULL) { - dError("failed to init dnode thread"); - return TSDB_CODE_DND_OUT_OF_MEMORY; + if (dndReadDnodes(pDnd) != 0) { + dError("failed to read file:%s since %s", pDnode->file, terrstr()); + return -1; } - int32_t code = dnodeReadDnodes(); - if (code != 0) { - dError("failed to read file:%s since %s", pDnode->file, tstrerror(code)); - return code; + pthread_mutex_init(&pDnode->mutex, NULL); + + pDnode->threadId = taosCreateThread(dnodeThreadRoutine, pDnd); + if (pDnode->threadId == NULL) { + dError("failed to init dnode thread"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - dInfo("dnode-dnode is initialized"); + dInfo("dnd-dnode is initialized"); return 0; } -void dnodeCleanupDnode(SDnode *pServer) { - SDnodeDnode *pDnode = &pServer->dnode; +void dndCleanupDnode(SDnode *pDnd) { + SDnodeMgmt *pDnode = &pDnd->d; if (pDnode->threadId != NULL) { - pDnode->threadStop = true; taosDestoryThread(pDnode->threadId); pDnode->threadId = NULL; } - pthread_mutex_lock(&pDnode->mutex); + dndLockDnode(pDnd); if (pDnode->dnodeEps != NULL) { free(pDnode->dnodeEps); pDnode->dnodeEps = NULL; } - if (pDnode->dnodeHash) { + if (pDnode->dnodeHash != NULL) { taosHashCleanup(pDnode->dnodeHash); pDnode->dnodeHash = NULL; } - pthread_mutex_unlock(&pDnode->mutex); + if (pDnode->file != NULL) { + free(pDnode->file); + pDnode->file = NULL; + } + + dndUnLockDnode(pDnd); pthread_mutex_destroy(&pDnode->mutex); - dInfo("dnode-dnode is cleaned up"); + dInfo("dnd-dnode is cleaned up"); } -void dnodeProcessDnodeMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - int32_t msgType = pMsg->msgType; - - if (msgType == TSDB_MSG_TYPE_STATUS_RSP && pEpSet) { - dnodeUpdateMnodeEpSet(&pDnode->dnode, pEpSet); - } - - switch (msgType) { +void dndProcessDnodeReq(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { + switch (pMsg->msgType) { case TSDB_MSG_TYPE_NETWORK_TEST: - dnodeProcessStartupReq(pMsg); + dndProcessStartupReq(pDnd, pMsg); break; case TSDB_MSG_TYPE_CONFIG_DNODE_IN: - dnodeProcessConfigDnodeReq(pMsg); - break; - case TSDB_MSG_TYPE_STATUS_RSP: - dnodeProcessStatusRsp(pMsg); + dndProcessConfigDnodeReq(pDnd, pMsg); break; default: - dError("RPC %p, %s not processed", pMsg->handle, taosMsg[msgType]); - SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_MSG_NOT_PROCESSED}; + dError("RPC %p, dnode req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED}; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); } } + +void dndProcessDnodeRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { + switch (pMsg->msgType) { + case TSDB_MSG_TYPE_STATUS_RSP: + dndProcessStatusRsp(pDnd, pMsg, pEpSet); + break; + case TSDB_MSG_TYPE_AUTH_RSP: + dndProcessAuthRsp(pDnd, pMsg, pEpSet); + break; + case TSDB_MSG_TYPE_GRANT_RSP: + dndProcessGrantRsp(pDnd, pMsg, pEpSet); + break; + default: + dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); + } +} diff --git a/source/dnode/mgmt/impl/src/dnodeInt.c b/source/dnode/mgmt/impl/src/dndInt.c similarity index 56% rename from source/dnode/mgmt/impl/src/dnodeInt.c rename to source/dnode/mgmt/impl/src/dndInt.c index 8641a54def..9e77da61b4 100644 --- a/source/dnode/mgmt/impl/src/dnodeInt.c +++ b/source/dnode/mgmt/impl/src/dndInt.c @@ -14,50 +14,47 @@ */ #define _DEFAULT_SOURCE -#include "dnodeDnode.h" -#include "dnodeMnode.h" -#include "dnodeTransport.h" -#include "dnodeVnodes.h" +#include "dndDnode.h" +#include "dndMnode.h" +#include "dndTransport.h" +#include "dndVnodes.h" #include "sync.h" #include "tcache.h" -#include "tconfig.h" -#include "tnote.h" -#include "tstep.h" #include "wal.h" -EStat dnodeGetStat(SDnode *pDnode) { return pDnode->stat; } +EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } -void dnodeSetStat(SDnode *pDnode, EStat stat) { - dDebug("dnode stat set from %s to %s", dnodeStatStr(pDnode->stat), dnodeStatStr(stat)); +void dndSetStat(SDnode *pDnode, EStat stat) { + dDebug("dnode stat set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat)); pDnode->stat = stat; } -char *dnodeStatStr(EStat stat) { +char *dndStatStr(EStat stat) { switch (stat) { - case DN_STAT_INIT: + case DND_STAT_INIT: return "init"; - case DN_STAT_RUNNING: + case DND_STAT_RUNNING: return "running"; - case DN_STAT_STOPPED: + case DND_STAT_STOPPED: return "stopped"; default: return "unknown"; } } -void dnodeReportStartup(SDnode *pDnode, char *name, char *desc) { +void dndReportStartup(SDnode *pDnode, char *name, char *desc) { SStartupMsg *pStartup = &pDnode->startup; tstrncpy(pStartup->name, name, strlen(pStartup->name)); tstrncpy(pStartup->desc, desc, strlen(pStartup->desc)); pStartup->finished = 0; } -void dnodeGetStartup(SDnode *pDnode, SStartupMsg *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupMsg); - pStartup->finished = (dnodeGetStat(pDnode) == DN_STAT_RUNNING); +void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupMsg)); + pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } -static int32_t dnodeCheckRunning(char *dataDir) { +static int32_t dndCheckRunning(char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); @@ -79,15 +76,19 @@ static int32_t dnodeCheckRunning(char *dataDir) { return 0; } -static int32_t dnodeInitDisk(SDnode *pDnode, char *dataDir) { - char path[PATH_MAX]; - snprintf(path, PATH_MAX, "%s/mnode", dataDir); +static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOptions) { + if (dndCheckRunning(pOptions->dataDir) != 0) { + return -1; + } + + char path[PATH_MAX + 100]; + snprintf(path, sizeof(path), "%s%smnode", pOptions->dataDir, TD_DIRSEP); pDnode->dir.mnode = strdup(path); - sprintf(path, PATH_MAX, "%s/vnode", dataDir); + snprintf(path, sizeof(path), "%s%svnode", pOptions->dataDir, TD_DIRSEP); pDnode->dir.vnodes = strdup(path); - sprintf(path, PATH_MAX, "%s/dnode", dataDir); + snprintf(path, sizeof(path), "%s%sdnode", pOptions->dataDir, TD_DIRSEP); pDnode->dir.dnode = strdup(path); if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL) { @@ -114,55 +115,10 @@ static int32_t dnodeInitDisk(SDnode *pDnode, char *dataDir) { return -1; } - if (dnodeCheckRunning(dataDir) != 0) { - return -1; - } - - return 0; -} - -static int32_t dnodeInitEnv(SDnode *pDnode, const char *cfgPath) { - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - taosSetCoreDump(tsEnableCoreFile); - - if (!taosMkDir(tsLogDir)) { - printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); - return -1; - } - - char temp[TSDB_FILENAME_LEN]; - sprintf(temp, "%s/taosdlog", tsLogDir); - if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) { - dError("failed to init log file\n"); - return -1; - } - - if (!taosReadGlobalCfg()) { - taosPrintGlobalCfg(); - dError("TDengine read global config failed"); - return -1; - } - - taosInitNotes(); - - if (taosCheckGlobalCfg() != 0) { - dError("TDengine check global config failed"); - return -1; - } - - if (dnodeInitDisk(pDnode, tsDataDir) != 0) { - dError("TDengine failed to init directory"); - return -1; - } - return 0; } -static void dnodeCleanupEnv(SDnode *pDnode) { +static void dndCleanupEnv(SDnode *pDnode) { if (pDnode->dir.mnode != NULL) { tfree(pDnode->dir.mnode); } @@ -175,11 +131,10 @@ static void dnodeCleanupEnv(SDnode *pDnode) { tfree(pDnode->dir.dnode); } - taosCloseLog(); taosStopCacheRefreshWorker(); } -SDnode *dnodeInit(const char *cfgPath) { +SDnode *dndInit(SDnodeOpt *pOptions) { SDnode *pDnode = calloc(1, sizeof(pDnode)); if (pDnode == NULL) { dError("failed to create dnode object"); @@ -188,73 +143,73 @@ SDnode *dnodeInit(const char *cfgPath) { } dInfo("start to initialize TDengine"); - dnodeSetStat(pDnode, DN_STAT_INIT); + dndSetStat(pDnode, DND_STAT_INIT); - if (dnodeInitEnv(pDnode, cfgPath) != 0) { + if (dndInitEnv(pDnode, pOptions) != 0) { dError("failed to init env"); - dnodeCleanup(pDnode); + dndCleanup(pDnode); return NULL; } if (rpcInit() != 0) { dError("failed to init rpc env"); - dnodeCleanup(pDnode); + dndCleanup(pDnode); return NULL; } if (walInit() != 0) { dError("failed to init wal env"); - dnodeCleanup(pDnode); + dndCleanup(pDnode); return NULL; } - if (dnodeInitDnode(pDnode) != 0) { + if (dndInitDnode(pDnode) != 0) { dError("failed to init dnode"); - dnodeCleanup(pDnode); + dndCleanup(pDnode); return NULL; } - if (dnodeInitVnodes(pDnode) != 0) { + if (dndInitVnodes(pDnode) != 0) { dError("failed to init vnodes"); - dnodeCleanup(pDnode); + dndCleanup(pDnode); return NULL; } - if (dnodeInitMnode(pDnode) != 0) { + if (dndInitMnode(pDnode) != 0) { dError("failed to init mnode"); - dnodeCleanup(pDnode); + dndCleanup(pDnode); return NULL; } - if (dnodeInitTrans(pDnode) != 0) { + if (dndInitTrans(pDnode) != 0) { dError("failed to init transport"); - dnodeCleanup(pDnode); + dndCleanup(pDnode); return NULL; } - dnodeSetStat(pDnode, DN_STAT_RUNNING); - dnodeReportStartup(pDnode, "TDengine", "initialized successfully"); + dndSetStat(pDnode, DND_STAT_RUNNING); + dndReportStartup(pDnode, "TDengine", "initialized successfully"); dInfo("TDengine is initialized successfully"); return 0; } -void dnodeCleanup(SDnode *pDnode) { - if (dnodeGetStat(pDnode) == DN_STAT_STOPPED) { +void dndCleanup(SDnode *pDnode) { + if (dndGetStat(pDnode) == DND_STAT_STOPPED) { dError("dnode is shutting down"); return; } dInfo("start to cleanup TDengine"); - dnodeSetStat(pDnode, DN_STAT_STOPPED); - dnodeCleanupTrans(pDnode); - dnodeCleanupMnode(pDnode); - dnodeCleanupVnodes(pDnode); - dnodeCleanupDnode(pDnode); + dndSetStat(pDnode, DND_STAT_STOPPED); + dndCleanupTrans(pDnode); + dndCleanupMnode(pDnode); + dndCleanupVnodes(pDnode); + dndCleanupDnode(pDnode); walCleanUp(); rpcCleanup(); dInfo("TDengine is cleaned up successfully"); - dnodeCleanupEnv(pDnode); + dndCleanupEnv(pDnode); free(pDnode); } diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c new file mode 100644 index 0000000000..5f3e48d8a1 --- /dev/null +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -0,0 +1,816 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndMnode.h" +#include "dndDnode.h" +#include "dndTransport.h" + +static int32_t dndInitMnodeReadWorker(SDnode *pDnode); +static int32_t dndInitMnodeWriteWorker(SDnode *pDnode); +static int32_t dndInitMnodeSyncWorker(SDnode *pDnode); +static int32_t dndInitMnodeMgmtWorker(SDnode *pDnode); +static void dndCleanupMnodeReadWorker(SDnode *pDnode); +static void dndCleanupMnodeWriteWorker(SDnode *pDnode); +static void dndCleanupMnodeSyncWorker(SDnode *pDnode); +static void dndCleanupMnodeMgmtWorker(SDnode *pDnode); +static int32_t dndAllocMnodeReadQueue(SDnode *pDnode); +static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode); +static int32_t dndAllocMnodeApplyQueue(SDnode *pDnode); +static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode); +static int32_t dndAllocMnodeMgmtQueue(SDnode *pDnode); +static void dndFreeMnodeReadQueue(SDnode *pDnode); +static void dndFreeMnodeWriteQueue(SDnode *pDnode); +static void dndFreeMnodeApplyQueue(SDnode *pDnode); +static void dndFreeMnodeSyncQueue(SDnode *pDnode); +static void dndFreeMnodeMgmtQueue(SDnode *pDnode); + +static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg); +static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg); +static void dndProcessMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg); +static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg); +static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); +static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg); +void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t dndPutMsgIntoMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg); + +static int32_t dndStartMnodeWorker(SDnode *pDnode); +static void dndStopMnodeWorker(SDnode *pDnode); + +static SMnode *dndAcquireMnode(SDnode *pDnode); +static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode); + +static int32_t dndReadMnodeFile(SDnode *pDnode); +static int32_t dndWriteMnodeFile(SDnode *pDnode); + +static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOptions *pOptions); +static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOptions *pOptions); +static int32_t dndDropMnode(SDnode *pDnode); + +static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); +static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); + +static SMnode *dndAcquireMnode(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + SMnode *pMnode = NULL; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pMgmt->deployed && !pMgmt->dropped) { + refCount = atomic_add_fetch_32(&pMgmt->refCount, 1); + pMnode = pMgmt->pMnode; + } else { + terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; + } + taosRUnLockLatch(&pMgmt->latch); + + dTrace("acquire mnode, refCount:%d", refCount); + return pMnode; +} + +static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pMnode != NULL) { + refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + dTrace("release mnode, refCount:%d", refCount); +} + +static int32_t dndReadMnodeFile(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + int32_t code = TSDB_CODE_DND_MNODE_READ_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 300; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + + FILE *fp = fopen(pMgmt->file, "r"); + if (fp == NULL) { + dDebug("file %s not exist", pMgmt->file); + code = 0; + goto PRASE_MNODE_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", pMgmt->file); + goto PRASE_MNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", pMgmt->file); + goto PRASE_MNODE_OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_String) { + dError("failed to read %s since deployed not found", pMgmt->file); + goto PRASE_MNODE_OVER; + } + pMgmt->deployed = atoi(deployed->valuestring); + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_String) { + dError("failed to read %s since dropped not found", pMgmt->file); + goto PRASE_MNODE_OVER; + } + pMgmt->dropped = atoi(dropped->valuestring); + + code = 0; + dInfo("succcessed to read file %s", pMgmt->file); + +PRASE_MNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + terrno = code; + return code; +} + +static int32_t dndWriteMnodeFile(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + char file[PATH_MAX + 20] = {0}; + snprintf(file, sizeof(file), "%s.bak", pMgmt->file); + + FILE *fp = fopen(file, "w"); + if (fp != NULL) { + terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 300; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": \"%d\",\n", pMgmt->deployed); + len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pMgmt->dropped); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFsyncFile(fileno(fp)); + fclose(fp); + free(content); + + if (taosRenameFile(file, pMgmt->file) != 0) { + terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; + dError("failed to rename %s since %s", pMgmt->file, terrstr()); + return -1; + } + + dInfo("successed to write %s", pMgmt->file); + return 0; +} + +static int32_t dndStartMnodeWorker(SDnode *pDnode) { + if (dndAllocMnodeReadQueue(pDnode) != 0) { + dError("failed to alloc mnode read queue since %s", terrstr()); + return -1; + } + + if (dndAllocMnodeWriteQueue(pDnode) != 0) { + dError("failed to alloc mnode write queue since %s", terrstr()); + return -1; + } + + if (dndAllocMnodeApplyQueue(pDnode) != 0) { + dError("failed to alloc mnode apply queue since %s", terrstr()); + return -1; + } + + if (dndAllocMnodeSyncQueue(pDnode) != 0) { + dError("failed to alloc mnode sync queue since %s", terrstr()); + return -1; + } + + return 0; +} + +static void dndStopMnodeWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + + taosWLockLatch(&pMgmt->latch); + pMgmt->deployed = 0; + pMgmt->pMnode = NULL; + taosWUnLockLatch(&pMgmt->latch); + + while (pMgmt->refCount > 1) taosMsleep(10); + while (!taosQueueEmpty(pMgmt->pReadQ)) taosMsleep(10); + while (!taosQueueEmpty(pMgmt->pApplyQ)) taosMsleep(10); + while (!taosQueueEmpty(pMgmt->pWriteQ)) taosMsleep(10); + while (!taosQueueEmpty(pMgmt->pSyncQ)) taosMsleep(10); + + dndFreeMnodeReadQueue(pDnode); + dndFreeMnodeWriteQueue(pDnode); + dndFreeMnodeApplyQueue(pDnode); + dndFreeMnodeSyncQueue(pDnode); + + dndCleanupMnodeReadWorker(pDnode); + dndCleanupMnodeWriteWorker(pDnode); + dndCleanupMnodeSyncWorker(pDnode); +} + +static bool dndNeedDeployMnode(SDnode *pDnode) { + if (dndGetDnodeId(pDnode) > 0) { + return false; + } + + if (dndGetClusterId(pDnode) > 0) { + return false; + } + if (strcmp(pDnode->opt.localEp, pDnode->opt.firstEp) != 0) { + return false; + } + + return true; +} + +static void dndInitMnodeOptions(SDnode *pDnode, SMnodeOptions *pOptions) { + pOptions->pDnode = pDnode; + pOptions->sendMsgToDnodeFp = dndSendMsgToDnode; + pOptions->sendMsgToMnodeFp = dndSendMsgToMnode; + pOptions->sendRedirectMsgFp = dndSendRedirectMsg; + pOptions->putMsgToApplyMsgFp = dndPutMsgIntoMnodeApplyQueue; +} + +static int32_t dndBuildMnodeOptions(SDnode *pDnode, SMnodeOptions *pOptions, SCreateMnodeMsg *pMsg) { + dndInitMnodeOptions(pDnode, pOptions); + + if (pMsg == NULL) { + pOptions->dnodeId = 1; + pOptions->clusterId = 1234; + pOptions->replica = 1; + pOptions->selfIndex = 0; + SReplica *pReplica = &pOptions->replicas[0]; + pReplica->id = 1; + pReplica->port = pDnode->opt.serverPort; + tstrncpy(pReplica->fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN); + } else { + pOptions->dnodeId = dndGetDnodeId(pDnode); + pOptions->clusterId = dndGetClusterId(pDnode); + pOptions->selfIndex = -1; + pOptions->replica = pMsg->replica; + for (int32_t index = 0; index < pMsg->replica; ++index) { + SReplica *pReplica = &pOptions->replicas[index]; + pReplica->id = pMsg->replicas[index].id; + pReplica->port = pMsg->replicas[index].port; + tstrncpy(pReplica->fqdn, pMsg->replicas[index].fqdn, TSDB_FQDN_LEN); + if (pReplica->id == pOptions->dnodeId) { + pOptions->selfIndex = index; + } + } + } + + if (pOptions->selfIndex == -1) { + terrno = TSDB_CODE_DND_MNODE_ID_NOT_FOUND; + dError("failed to build mnode options since %s", terrstr()); + return -1; + } + + return 0; +} + +static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOptions *pOptions) { + SMnodeMgmt *pMgmt = &pDnode->m; + + int32_t code = dndStartMnodeWorker(pDnode); + if (code != 0) { + dError("failed to start mnode worker since %s", terrstr()); + return code; + } + + SMnode *pMnode = mnodeOpen(pDnode->dir.mnode, pOptions); + if (pMnode == NULL) { + dError("failed to open mnode since %s", terrstr()); + code = terrno; + dndStopMnodeWorker(pDnode); + terrno = code; + return code; + } + + if (dndWriteMnodeFile(pDnode) != 0) { + dError("failed to write mnode file since %s", terrstr()); + code = terrno; + dndStopMnodeWorker(pDnode); + mnodeClose(pMnode); + mnodeDestroy(pDnode->dir.mnode); + terrno = code; + return code; + } + + taosWLockLatch(&pMgmt->latch); + pMgmt->pMnode = pMnode; + pMgmt->deployed = 1; + taosWUnLockLatch(&pMgmt->latch); + + return 0; +} + +static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOptions *pOptions) { + SMnodeMgmt *pMgmt = &pDnode->m; + + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL) { + dError("failed to alter mnode since %s", terrstr()); + return -1; + } + + if (mnodeAlter(pMnode, pOptions) != 0) { + dError("failed to alter mnode since %s", terrstr()); + dndReleaseMnode(pDnode, pMnode); + return -1; + } + + dndReleaseMnode(pDnode, pMnode); + return 0; +} + +static int32_t dndDropMnode(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL) { + dError("failed to drop mnode since %s", terrstr()); + return -1; + } + + taosRLockLatch(&pMgmt->latch); + pMgmt->dropped = 1; + taosRUnLockLatch(&pMgmt->latch); + + if (dndWriteMnodeFile(pDnode) != 0) { + taosRLockLatch(&pMgmt->latch); + pMgmt->dropped = 0; + taosRUnLockLatch(&pMgmt->latch); + + dndReleaseMnode(pDnode, pMnode); + dError("failed to drop mnode since %s", terrstr()); + return -1; + } + + dndStopMnodeWorker(pDnode); + dndWriteMnodeFile(pDnode); + mnodeClose(pMnode); + mnodeDestroy(pDnode->dir.mnode); + + return 0; +} + +static SCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { + SCreateMnodeMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); + for (int32_t i = 0; i < pMsg->replica; ++i) { + pMsg->replicas[i].id = htonl(pMsg->replicas[i].id); + pMsg->replicas[i].port = htons(pMsg->replicas[i].port); + } + + return pMsg; +} + +static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SCreateMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_MNODE_ID_INVALID; + return -1; + } else { + SMnodeOptions option = {0}; + if (dndBuildMnodeOptions(pDnode, &option, pMsg) != 0) { + return -1; + } + return dndOpenMnode(pDnode, &option); + } +} + +static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SAlterMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_MNODE_ID_INVALID; + return -1; + } else { + SMnodeOptions option = {0}; + if (dndBuildMnodeOptions(pDnode, &option, pMsg) != 0) { + return -1; + } + return dndAlterMnode(pDnode, &option); + } +} + +static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { + SDropMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + + if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_MNODE_ID_INVALID; + return -1; + } else { + return dndDropMnode(pDnode); + } +} + +static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { + int32_t code = 0; + + switch (pMsg->msgType) { + case TSDB_MSG_TYPE_CREATE_MNODE_IN: + code = dndProcessCreateMnodeReq(pDnode, pMsg); + break; + case TSDB_MSG_TYPE_ALTER_MNODE_IN: + code = dndProcessAlterMnodeReq(pDnode, pMsg); + break; + case TSDB_MSG_TYPE_DROP_MNODE_IN: + code = dndProcessDropMnodeReq(pDnode, pMsg); + break; + default: + code = TSDB_CODE_MSG_NOT_PROCESSED; + break; + } + + SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); +} + +static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) { + SMnodeMgmt *pMgmt = &pDnode->m; + + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode != NULL) { + mnodeProcessReadMsg(pMnode, pMsg); + dndReleaseMnode(pDnode, pMnode); + } else { + mnodeSendRsp(pMsg, terrno); + } + + mnodeCleanupMsg(pMsg); +} + +static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg) { + SMnodeMgmt *pMgmt = &pDnode->m; + + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode != NULL) { + mnodeProcessWriteMsg(pMnode, pMsg); + dndReleaseMnode(pDnode, pMnode); + } else { + mnodeSendRsp(pMsg, terrno); + } + + mnodeCleanupMsg(pMsg); +} + +static void dndProcessMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg) { + SMnodeMgmt *pMgmt = &pDnode->m; + + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode != NULL) { + mnodeProcessApplyMsg(pMnode, pMsg); + dndReleaseMnode(pDnode, pMnode); + } else { + mnodeSendRsp(pMsg, terrno); + } + + mnodeCleanupMsg(pMsg); +} + +static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg) { + SMnodeMgmt *pMgmt = &pDnode->m; + + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode != NULL) { + mnodeProcessSyncMsg(pMnode, pMsg); + dndReleaseMnode(pDnode, pMnode); + } else { + mnodeSendRsp(pMsg, terrno); + } + + mnodeCleanupMsg(pMsg); +} + +static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg) { + assert(pQueue); + + SMnodeMsg *pMsg = mnodeInitMsg(pMnode, pRpcMsg); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (taosWriteQitem(pQueue, pMsg) != 0) { + mnodeCleanupMsg(pMsg); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet) { + SMnodeMgmt *pMgmt = &pDnode->m; + SMnode *pMnode = dndAcquireMnode(pDnode); + + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); + if (pMsg == NULL || taosWriteQitem(pMgmt->pMgmtQ, pMsg) != 0) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; + rpcSendResponse(&rsp); + rpcFreeCont(pRpcMsg->pCont); + taosFreeQitem(pMsg); + } +} + +void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SMnodeMgmt *pMgmt = &pDnode->m; + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + } + + dndReleaseMnode(pDnode, pMnode); +} + +void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SMnodeMgmt *pMgmt = &pDnode->m; + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + } + + dndReleaseMnode(pDnode, pMnode); +} + +void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SMnodeMgmt *pMgmt = &pDnode->m; + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + } + + dndReleaseMnode(pDnode, pMnode); +} + +static int32_t dndPutMsgIntoMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg) { + SMnodeMgmt *pMgmt = &pDnode->m; + + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL) { + return -1; + } + + int32_t code = taosWriteQitem(pMgmt->pApplyQ, pMsg); + dndReleaseMnode(pDnode, pMnode); + return code; +} + +static int32_t dndAllocMnodeMgmtQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + pMgmt->pMgmtQ = tWorkerAllocQueue(&pMgmt->mgmtPool, NULL, (FProcessItem)dndProcessMnodeMgmtQueue); + if (pMgmt->pMgmtQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndFreeMnodeMgmtQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + tWorkerFreeQueue(&pMgmt->mgmtPool, pMgmt->pMgmtQ); + pMgmt->pMgmtQ = NULL; +} + +static int32_t dndInitMnodeMgmtWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + SWorkerPool *pPool = &pMgmt->mgmtPool; + pPool->name = "mnode-mgmt"; + pPool->min = 1; + pPool->max = 1; + if (tWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndCleanupMnodeMgmtWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + ; + tWorkerCleanup(&pMgmt->mgmtPool); +} + +static int32_t dndAllocMnodeReadQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + pMgmt->pReadQ = tWorkerAllocQueue(&pMgmt->readPool, NULL, (FProcessItem)dndProcessMnodeReadQueue); + if (pMgmt->pReadQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndFreeMnodeReadQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + tWorkerFreeQueue(&pMgmt->readPool, pMgmt->pReadQ); + pMgmt->pReadQ = NULL; +} + +static int32_t dndInitMnodeReadWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + SWorkerPool *pPool = &pMgmt->readPool; + pPool->name = "mnode-read"; + pPool->min = 0; + pPool->max = 1; + if (tWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndCleanupMnodeReadWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + tWorkerCleanup(&pMgmt->readPool); +} + +static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + pMgmt->pWriteQ = tWorkerAllocQueue(&pMgmt->writePool, NULL, (FProcessItem)dndProcessMnodeWriteQueue); + if (pMgmt->pWriteQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndFreeMnodeWriteQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pWriteQ); + pMgmt->pWriteQ = NULL; +} + +static int32_t dndAllocMnodeApplyQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + pMgmt->pApplyQ = tWorkerAllocQueue(&pMgmt->writePool, NULL, (FProcessItem)dndProcessMnodeApplyQueue); + if (pMgmt->pApplyQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndFreeMnodeApplyQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pApplyQ); + pMgmt->pApplyQ = NULL; +} + +static int32_t dndInitMnodeWriteWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + SWorkerPool *pPool = &pMgmt->writePool; + pPool->name = "mnode-write"; + pPool->min = 0; + pPool->max = 1; + if (tWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndCleanupMnodeWriteWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + tWorkerCleanup(&pMgmt->writePool); +} + +static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + pMgmt->pSyncQ = tWorkerAllocQueue(&pMgmt->syncPool, NULL, (FProcessItem)dndProcessMnodeSyncQueue); + if (pMgmt->pSyncQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndFreeMnodeSyncQueue(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + tWorkerFreeQueue(&pMgmt->syncPool, pMgmt->pSyncQ); + pMgmt->pSyncQ = NULL; +} + +static int32_t dndInitMnodeSyncWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + SWorkerPool *pPool = &pMgmt->syncPool; + pPool->name = "mnode-sync"; + pPool->min = 0; + pPool->max = 1; + return tWorkerInit(pPool); +} + +static void dndCleanupMnodeSyncWorker(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + tWorkerCleanup(&pMgmt->syncPool); +} + +int32_t dndInitMnode(SDnode *pDnode) { + dInfo("dnode-mnode start to init"); + SMnodeMgmt *pMgmt = &pDnode->m; + taosInitRWLatch(&pMgmt->latch); + + if (dndInitMnodeMgmtWorker(pDnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + char path[PATH_MAX]; + snprintf(path, PATH_MAX, "%s/mnode.json", pDnode->dir.dnode); + pMgmt->file = strdup(path); + if (pMgmt->file == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (dndReadMnodeFile(pDnode) != 0) { + return -1; + } + + if (pMgmt->dropped) { + dInfo("mnode has been deployed and needs to be deleted"); + mnodeDestroy(pDnode->dir.mnode); + return 0; + } + + if (!pMgmt->deployed) { + bool needDeploy = dndNeedDeployMnode(pDnode); + if (!needDeploy) { + dDebug("mnode does not need to be deployed"); + return 0; + } + + dInfo("start to deploy mnode"); + } else { + dInfo("start to open mnode"); + } + + SMnodeOptions option = {0}; + dndInitMnodeOptions(pDnode, &option); + return dndOpenMnode(pDnode, &option); +} + +void dndCleanupMnode(SDnode *pDnode) { + SMnodeMgmt *pMgmt = &pDnode->m; + + dInfo("dnode-mnode start to clean up"); + dndStopMnodeWorker(pDnode); + dndCleanupMnodeMgmtWorker(pDnode); + tfree(pMgmt->file); + dInfo("dnode-mnode is cleaned up"); +} + +int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SMnodeMgmt *pMgmt = &pDnode->m; + + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL) { + terrno = TSDB_CODE_APP_NOT_READY; + dTrace("failed to get user auth since %s", terrstr()); + return -1; + } + + int32_t code = mnodeRetriveAuth(pMnode, user, spi, encrypt, secret, ckey); + dndReleaseMnode(pDnode, pMnode); + return code; +} diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c new file mode 100644 index 0000000000..679e3ef5f9 --- /dev/null +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/* this file is mainly responsible for the communication between DNODEs. Each + * dnode works as both server and client. Dnode may send status, grant, config + * messages to mnode, mnode may send create/alter/drop table/vnode messages + * to dnode. All theses messages are handled from here + */ + +#define _DEFAULT_SOURCE +#include "dndTransport.h" +#include "dndDnode.h" +#include "dndMnode.h" +#include "dndVnodes.h" + +static void dndInitMsgFp(STransMgmt *pMgmt) { + // msg from client to dnode + pMgmt->msgFp[TSDB_MSG_TYPE_SUBMIT] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_QUERY] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_FETCH] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TABLE] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_TABLE_META] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_TABLES_META] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_ACK] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_RESET] = dndProcessVnodeWriteMsg; + + // msg from client to mnode + pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_ACCT] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_ACCT] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_ACCT] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_USER] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_USER] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_USER] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_DNODE] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_DNODE] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_DB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_DB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_USE_DB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_DB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_DB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TOPIC] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_FUNCTION] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dndProcessMnodeWriteMsg; + + // message from client to dnode + pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq; + + // message from mnode to vnode + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; + + // message from mnode to dnode + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN] = dndProcessVnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN] = dndProcessMnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN] = dndProcessMnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN] = dndProcessMnodeMgmtMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN] = dndProcessDnodeReq; + pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP] = dndProcessMnodeWriteMsg; + + // message from dnode to mnode + pMgmt->msgFp[TSDB_MSG_TYPE_AUTH] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_RSP] = dndProcessDnodeRsp; + pMgmt->msgFp[TSDB_MSG_TYPE_GRANT] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_GRANT_RSP] = dndProcessDnodeRsp; + pMgmt->msgFp[TSDB_MSG_TYPE_STATUS] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_STATUS_RSP] = dndProcessDnodeRsp; +} + +static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SDnode *pDnode = parent; + STransMgmt *pMgmt = &pDnode->t; + + int32_t msgType = pMsg->msgType; + + if (dndGetStat(pDnode) == DND_STAT_STOPPED) { + if (pMsg == NULL || pMsg->pCont == NULL) return; + dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]); + rpcFreeCont(pMsg->pCont); + return; + } + + DndMsgFp fp = pMgmt->msgFp[msgType]; + if (fp != NULL) { + dTrace("RPC %p, rsp:%s will be processed, code:%s", pMsg->handle, taosMsg[msgType], tstrerror(pMsg->code)); + (*fp)(pDnode, pMsg, pEpSet); + } else { + dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]); + rpcFreeCont(pMsg->pCont); + } +} + +static int32_t dndInitClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->t; + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND-C"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = dndProcessResponse; + rpcInit.sessions = TSDB_MAX_VNODES << 4; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000; + rpcInit.user = "-internal"; + rpcInit.ckey = "-key"; + rpcInit.secret = "-secret"; + + pMgmt->clientRpc = rpcOpen(&rpcInit); + if (pMgmt->clientRpc == NULL) { + dError("failed to init rpc client"); + return -1; + } + + return 0; +} + +static void dndCleanupClient(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->t; + if (pMgmt->clientRpc) { + rpcClose(pMgmt->clientRpc); + pMgmt->clientRpc = NULL; + dInfo("dnode peer rpc client is closed"); + } +} + +static void dndProcessRequest(void *param, SRpcMsg *pMsg, SEpSet *pEpSet) { + SDnode *pDnode = param; + STransMgmt *pMgmt = &pDnode->t; + + int32_t msgType = pMsg->msgType; + if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + dndProcessDnodeReq(pDnode, pMsg, pEpSet); + return; + } + + if (dndGetStat(pDnode) == DND_STAT_STOPPED) { + dError("RPC %p, req:%s is ignored since dnode exiting", pMsg->handle, taosMsg[msgType]); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_EXITING}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); + return; + } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) { + dError("RPC %p, req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); + return; + } + + if (pMsg->pCont == NULL) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN}; + rpcSendResponse(&rspMsg); + return; + } + + DndMsgFp fp = pMgmt->msgFp[msgType]; + if (fp != NULL) { + dTrace("RPC %p, req:%s will be processed", pMsg->handle, taosMsg[msgType]); + (*fp)(pDnode, pMsg, pEpSet); + } else { + dError("RPC %p, req:%s is not processed", pMsg->handle, taosMsg[msgType]); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); + } +} + +static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { + STransMgmt *pMgmt = &pDnode->t; + + SEpSet epSet = {0}; + dndGetMnodeEpSet(pDnode, &epSet); + rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); +} + +static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SDnode *pDnode = parent; + + if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) != 0) { + if (terrno != TSDB_CODE_APP_NOT_READY) { + dTrace("failed to get user auth from mnode since %s", terrstr()); + return -1; + } + } + + dDebug("user:%s, send auth msg to mnodes", user); + + SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); + tstrncpy(pMsg->user, user, TSDB_USER_LEN); + + SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TSDB_MSG_TYPE_AUTH}; + SRpcMsg rpcRsp = {0}; + dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + terrno = rpcRsp.code; + dError("user:%s, failed to get user auth from mnodes since %s", user, terrstr()); + } else { + SAuthRsp *pRsp = rpcRsp.pCont; + memcpy(secret, pRsp->secret, TSDB_KEY_LEN); + memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); + *spi = pRsp->spi; + *encrypt = pRsp->encrypt; + dDebug("user:%s, success to get user auth from mnodes", user); + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; +} + +static int32_t dndInitServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->t; + dndInitMsgFp(pMgmt); + + int32_t numOfThreads = (int32_t)((pDnode->opt.numOfCores * pDnode->opt.numOfThreadsPerCore) / 2.0); + if (numOfThreads < 1) { + numOfThreads = 1; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = pDnode->opt.serverPort; + rpcInit.label = "DND-S"; + rpcInit.numOfThreads = numOfThreads; + rpcInit.cfp = dndProcessRequest; + rpcInit.sessions = pDnode->opt.maxShellConns; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000; + rpcInit.afp = dndRetrieveUserAuthInfo; + + pMgmt->serverRpc = rpcOpen(&rpcInit); + if (pMgmt->serverRpc == NULL) { + dError("failed to init rpc server"); + return -1; + } + + return 0; +} + +static void dndCleanupServer(SDnode *pDnode) { + STransMgmt *pMgmt = &pDnode->t; + if (pMgmt->serverRpc) { + rpcClose(pMgmt->serverRpc); + pMgmt->serverRpc = NULL; + } +} + +int32_t dndInitTrans(SDnode *pDnode) { + if (dndInitClient(pDnode) != 0) { + return -1; + } + + if (dndInitServer(pDnode) != 0) { + return -1; + } + + dInfo("dnode-transport is initialized"); + return 0; +} + +void dndCleanupTrans(SDnode *pDnode) { + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); + dInfo("dnode-transport is cleaned up"); +} + +void dndSendMsgToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pMsg) { + STransMgmt *pMgmt = &pDnode->t; + rpcSendRequest(pMgmt->clientRpc, pEpSet, pMsg, NULL); +} + +void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pMsg) { + SEpSet epSet = {0}; + dndGetMnodeEpSet(pDnode, &epSet); + dndSendMsgToDnode(pDnode, &epSet, pMsg); +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c new file mode 100644 index 0000000000..d5e94106a7 --- /dev/null +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -0,0 +1,1119 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dndVnodes.h" +#include "dndTransport.h" + +typedef struct { + int32_t vgId; + int32_t refCount; + int8_t dropped; + int8_t accessState; + SVnode *pImpl; + taos_queue pWriteQ; + taos_queue pSyncQ; + taos_queue pApplyQ; + taos_queue pQueryQ; + taos_queue pFetchQ; +} SVnodeObj; + +typedef struct { + int32_t vnodeNum; + int32_t opened; + int32_t failed; + int32_t threadIndex; + pthread_t *pThreadId; + SVnodeObj *pVnodes; + SDnode *pDnode; +} SVnodeThread; + +static int32_t dndInitVnodeReadWorker(SDnode *pDnode); +static int32_t dndInitVnodeWriteWorker(SDnode *pDnode); +static int32_t dndInitVnodeSyncWorker(SDnode *pDnode); +static int32_t dndInitVnodeMgmtWorker(SDnode *pDnode); +static void dndCleanupVnodeReadWorker(SDnode *pDnode); +static void dndCleanupVnodeWriteWorker(SDnode *pDnode); +static void dndCleanupVnodeSyncWorker(SDnode *pDnode); +static void dndCleanupVnodeMgmtWorker(SDnode *pDnode); +static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); +static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode); +static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode); +static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode); +static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode); +static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); + +static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg); +static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg); +static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); +static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); +static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); +static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); +void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg); + +static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId); +static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, SVnode *pImpl); +static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode); +static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes); +static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes); +static int32_t dndWriteVnodesToFile(SDnode *pDnode); + +static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg); +static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndOpenVnodes(SDnode *pDnode); +static void dndCloseVnodes(SDnode *pDnode); + +static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); +static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); +static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); +static int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); +static int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); +static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); + +static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + SVnodeObj *pVnode = NULL; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + if (pVnode == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + } else { + refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + return pVnode; +} + +static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + int32_t refCount = 0; + + taosRLockLatch(&pMgmt->latch); + if (pVnode != NULL) { + refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + } + taosRUnLockLatch(&pMgmt->latch); + + if (pVnode != NULL) { + dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); + } +} + +static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, SVnode *pImpl) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); + if (pVnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pVnode->vgId = vgId; + pVnode->refCount = 0; + pVnode->dropped = 0; + pVnode->accessState = TSDB_VN_ALL_ACCCESS; + pVnode->pImpl = pImpl; + + if (dndAllocVnodeQueryQueue(pDnode, pVnode) != 0) { + return -1; + } + + if (dndAllocVnodeFetchQueue(pDnode, pVnode) != 0) { + return -1; + } + + if (dndAllocVnodeWriteQueue(pDnode, pVnode) != 0) { + return -1; + } + + if (dndAllocVnodeApplyQueue(pDnode, pVnode) != 0) { + return -1; + } + + if (dndAllocVnodeSyncQueue(pDnode, pVnode) != 0) { + return -1; + } + + taosWLockLatch(&pMgmt->latch); + int32_t code = taosHashPut(pMgmt->hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); + taosWUnLockLatch(&pMgmt->latch); + + if (code != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return code; +} + +static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + taosWLockLatch(&pMgmt->latch); + taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); + taosWUnLockLatch(&pMgmt->latch); + + dndReleaseVnode(pDnode, pVnode); + while (pVnode->refCount > 0) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + + dndFreeVnodeQueryQueue(pDnode, pVnode); + dndFreeVnodeFetchQueue(pDnode, pVnode); + dndFreeVnodeWriteQueue(pDnode, pVnode); + dndFreeVnodeApplyQueue(pDnode, pVnode); + dndFreeVnodeSyncQueue(pDnode, pVnode); +} + +static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + taosRLockLatch(&pMgmt->latch); + + int32_t num = 0; + int32_t size = taosHashGetSize(pMgmt->hash); + SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + SVnodeObj *pVnode = *ppVnode; + if (pVnode) { + num++; + if (num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + pVnodes[num] = (*ppVnode); + } + } + pIter = taosHashIterate(pMgmt->hash, pIter); + } + + taosRUnLockLatch(&pMgmt->latch); + *numOfVnodes = num; + + return pVnodes; +} + +static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes) { + int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 30000; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; + char file[PATH_MAX + 20] = {0}; + SVnodeObj *pVnodes = NULL; + + snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); + + fp = fopen(file, "r"); + if (!fp) { + dDebug("file %s not exist", file); + code = 0; + goto PRASE_VNODE_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", file); + goto PRASE_VNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", file); + goto PRASE_VNODE_OVER; + } + + cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); + if (!vnodes || vnodes->type != cJSON_Array) { + dError("failed to read %s since vnodes not found", file); + goto PRASE_VNODE_OVER; + } + + int32_t vnodesNum = cJSON_GetArraySize(vnodes); + if (vnodesNum <= 0) { + dError("failed to read %s since vnodes size:%d invalid", file, vnodesNum); + goto PRASE_VNODE_OVER; + } + + pVnodes = calloc(vnodesNum, sizeof(SVnodeObj)); + if (pVnodes == NULL) { + dError("failed to read %s since out of memory", file); + goto PRASE_VNODE_OVER; + } + + for (int32_t i = 0; i < vnodesNum; ++i) { + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + SVnodeObj *pVnode = &pVnodes[i]; + + cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); + if (!vgId || vgId->type != cJSON_String) { + dError("failed to read %s since vgId not found", file); + goto PRASE_VNODE_OVER; + } + pVnode->vgId = atoi(vgId->valuestring); + + cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); + if (!dropped || dropped->type != cJSON_String) { + dError("failed to read %s since dropped not found", file); + goto PRASE_VNODE_OVER; + } + pVnode->dropped = atoi(vnode->valuestring); + } + + code = 0; + dInfo("succcessed to read file %s", file); + +PRASE_VNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + return code; +} + +static int32_t dndWriteVnodesToFile(SDnode *pDnode) { + char file[PATH_MAX + 20] = {0}; + char realfile[PATH_MAX + 20] = {0}; + snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); + snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); + + FILE *fp = fopen(file, "w"); + if (fp != NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 30000; + char *content = calloc(1, maxLen + 1); + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n"); + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId); + len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped); + if (i < numOfVnodes - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFsyncFile(fileno(fp)); + fclose(fp); + free(content); + terrno = 0; + + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = pVnodes[i]; + dndReleaseVnode(pDnode, pVnode); + } + + if (pVnodes != NULL) { + free(pVnodes); + } + + dInfo("successed to write %s", file); + return taosRenameFile(file, realfile); +} + +static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg) { + char path[PATH_MAX + 20] = {0}; + snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, vgId); + SVnode *pImpl = vnodeCreate(vgId, path, pCfg); + + if (pImpl == NULL) { + return -1; + } + + int32_t code = dndCreateVnodeWrapper(pDnode, vgId, pImpl); + if (code != 0) { + vnodeDrop(pImpl); + terrno = code; + return code; + } + + code = dndWriteVnodesToFile(pDnode); + if (code != 0) { + vnodeDrop(pImpl); + terrno = code; + return code; + } + + return 0; +} + +static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode) { + pVnode->dropped = 1; + if (dndWriteVnodesToFile(pDnode) != 0) { + pVnode->dropped = 0; + return -1; + } + + dndDropVnodeWrapper(pDnode, pVnode); + vnodeDrop(pVnode->pImpl); + dndWriteVnodesToFile(pDnode); + return 0; +} + +static void *dnodeOpenVnodeFunc(void *param) { + SVnodeThread *pThread = param; + SDnode *pDnode = pThread->pDnode; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + + dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + setThreadName("open-vnodes"); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + SVnodeObj *pVnode = &pThread->pVnodes[v]; + + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId, + pMgmt->openVnodes, pMgmt->totalVnodes); + dndReportStartup(pDnode, "open-vnodes", stepDesc); + + char path[PATH_MAX + 20] = {0}; + snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, pVnode->vgId); + SVnode *pImpl = vnodeOpen(path, NULL); + if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex); + pThread->failed++; + } else { + dndCreateVnodeWrapper(pDnode, pVnode->vgId, pImpl); + dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex); + pThread->opened++; + } + + atomic_add_fetch_32(&pMgmt->openVnodes, 1); + } + + dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t dndOpenVnodes(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + taosInitRWLatch(&pMgmt->latch); + + pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMgmt->hash == NULL) { + dError("failed to init vnode hash"); + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + SVnodeObj *pVnodes = NULL; + int32_t numOfVnodes = 0; + if (dndGetVnodesFromFile(pDnode, &pVnodes, &numOfVnodes) != 0) { + dInfo("failed to get vnode list from disk since %s", terrstr()); + return -1; + } + + pMgmt->totalVnodes = numOfVnodes; + + int32_t threadNum = tsNumOfCores; + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + + SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SVnodeThread *pThread = &threads[t]; + pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v]; + } + + dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + pThread->pThreadId = taosCreateThread(dnodeOpenVnodeFunc, pThread); + if (pThread->pThreadId == NULL) { + dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + } + } + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + taosDestoryThread(pThread->pThreadId); + pThread->pThreadId = NULL; + free(pThread->pVnodes); + } + free(threads); + + if (pMgmt->openVnodes != pMgmt->totalVnodes) { + dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes); + return -1; + } else { + dInfo("total vnodes:%d open successfully", pMgmt->totalVnodes); + return 0; + } +} + +static void dndCloseVnodes(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + + int32_t numOfVnodes = 0; + SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); + + for (int32_t i = 0; i < numOfVnodes; ++i) { + dndDropVnodeWrapper(pDnode, pVnodes[i]); + } + + if (pVnodes != NULL) { + free(pVnodes); + } + + if (pMgmt->hash != NULL) { + taosHashCleanup(pMgmt->hash); + pMgmt->hash = NULL; + } + + dInfo("total vnodes:%d are all closed", numOfVnodes); +} + +static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) { + SCreateVnodeMsg *pCreate = rpcMsg->pCont; + *vgId = htonl(pCreate->vgId); + + tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN); + pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize); + pCfg->totalBlocks = htonl(pCreate->totalBlocks); + pCfg->daysPerFile = htonl(pCreate->daysPerFile); + pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0); + pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1); + pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2); + pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); + pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + pCfg->precision = pCreate->precision; + pCfg->compression = pCreate->compression; + pCfg->cacheLastRow = pCreate->cacheLastRow; + pCfg->update = pCreate->update; + pCfg->quorum = pCreate->quorum; + pCfg->replica = pCreate->replica; + pCfg->walLevel = pCreate->walLevel; + pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod); + + for (int32_t i = 0; i < pCfg->replica; ++i) { + pCfg->replicas[i].port = htons(pCreate->replicas[i].port); + tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); + } + + return 0; +} + +static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) { + SDropVnodeMsg *pDrop = rpcMsg->pCont; + pDrop->vgId = htonl(pDrop->vgId); + return pDrop; +} + +static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) { + SAuthVnodeMsg *pAuth = rpcMsg->pCont; + pAuth->vgId = htonl(pAuth->vgId); + return pAuth; +} + +static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SVnodeCfg vnodeCfg = {0}; + int32_t vgId = 0; + + dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); + dDebug("vgId:%d, create vnode req is received", vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + if (pVnode != NULL) { + dDebug("vgId:%d, already exist, return success", vgId); + dndReleaseVnode(pDnode, pVnode); + return 0; + } + + if (dndCreateVnode(pDnode, vgId, &vnodeCfg) != 0) { + dError("vgId:%d, failed to create vnode since %s", vgId, terrstr()); + return terrno; + } + + return 0; +} + +static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SVnodeCfg vnodeCfg = {0}; + int32_t vgId = 0; + + dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); + dDebug("vgId:%d, alter vnode req is received", vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); + return terrno; + } + + if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { + dError("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); + dndReleaseVnode(pDnode, pVnode); + return terrno; + } + + dndReleaseVnode(pDnode, pVnode); + return 0; +} + +static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SDropVnodeMsg *pDrop = vnodeParseDropVnodeReq(rpcMsg); + + int32_t vgId = pDrop->vgId; + dDebug("vgId:%d, drop vnode req is received", vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); + return terrno; + } + + if (dndDropVnode(pDnode, pVnode) != 0) { + dError("vgId:%d, failed to drop vnode since %s", vgId, terrstr()); + dndReleaseVnode(pDnode, pVnode); + return terrno; + } + + return 0; +} + +static int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg); + + int32_t code = 0; + int32_t vgId = pAuth->vgId; + dDebug("vgId:%d, auth vnode req is received", vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to auth since %s", vgId, terrstr()); + return terrno; + } + + pVnode->accessState = pAuth->accessState; + dndReleaseVnode(pDnode, pVnode); + return 0; +} + +static int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg); + + int32_t vgId = pAuth->vgId; + dDebug("vgId:%d, auth vnode req is received", vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to auth since %s", vgId, terrstr()); + return terrno; + } + + if (vnodeSync(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to auth vnode since %s", vgId, terrstr()); + dndReleaseVnode(pDnode, pVnode); + return terrno; + } + + dndReleaseVnode(pDnode, pVnode); + return 0; +} + +static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SCompactVnodeMsg *pCompact = (SCompactVnodeMsg *)vnodeParseDropVnodeReq(rpcMsg); + + int32_t vgId = pCompact->vgId; + dDebug("vgId:%d, compact vnode req is received", vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); + return terrno; + } + + if (vnodeCompact(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); + dndReleaseVnode(pDnode, pVnode); + return terrno; + } + + dndReleaseVnode(pDnode, pVnode); + return 0; +} + +static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { + int32_t code = 0; + + switch (pMsg->msgType) { + case TSDB_MSG_TYPE_CREATE_VNODE_IN: + code = dndProcessCreateVnodeReq(pDnode, pMsg); + break; + case TSDB_MSG_TYPE_ALTER_VNODE_IN: + code = dndProcessAlterVnodeReq(pDnode, pMsg); + break; + case TSDB_MSG_TYPE_DROP_VNODE_IN: + code = dndProcessDropVnodeReq(pDnode, pMsg); + break; + case TSDB_MSG_TYPE_AUTH_VNODE_IN: + code = dndProcessAuthVnodeReq(pDnode, pMsg); + break; + case TSDB_MSG_TYPE_SYNC_VNODE_IN: + code = dndProcessSyncVnodeReq(pDnode, pMsg); + break; + case TSDB_MSG_TYPE_COMPACT_VNODE_IN: + code = dndProcessCompactVnodeReq(pDnode, pMsg); + break; + default: + code = TSDB_CODE_MSG_NOT_PROCESSED; + break; + } + + if (code != 0) { + SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } +} + +static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { + vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY); +} + +static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { + vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH); +} + +static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { + SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs); + SRpcMsg *pRpcMsg = NULL; + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + vnodeAppendMsg(pMsg, pRpcMsg); + taosFreeQitem(pRpcMsg); + } + + vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE); +} + +static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { + SVnodeMsg *pMsg = NULL; + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY); + } +} + +static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { + SVnodeMsg *pMsg = NULL; + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC); + } +} + +static int32_t dndWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { + int32_t code = 0; + + if (pQueue == NULL) { + code = TSDB_CODE_MSG_NOT_PROCESSED; + } else { + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); + if (pMsg == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + *pMsg = *pRpcMsg; + if (taosWriteQitem(pQueue, pMsg) != 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + } + + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; + rpcSendResponse(&rsp); + rpcFreeCont(pRpcMsg->pCont); + } +} + +static int32_t dndWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { + int32_t code = 0; + + if (pQueue == NULL) { + code = TSDB_CODE_MSG_NOT_PROCESSED; + } else { + SVnodeMsg *pMsg = vnodeInitMsg(1); + if (pMsg == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + if (vnodeAppendMsg(pMsg, pRpcMsg) != 0) { + code = terrno; + } else { + if (taosWriteQitem(pQueue, pMsg) != 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + } + } + + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; + rpcSendResponse(&rsp); + rpcFreeCont(pRpcMsg->pCont); + } +} + +static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { + SMsgHead *pHead = (SMsgHead *)pMsg->pCont; + pHead->vgId = htonl(pHead->vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); + if (pVnode == NULL) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + } + + return pVnode; +} + +void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + dndWriteRpcMsgToVnodeQueue(pMgmt->pMgmtQ, pMsg); +} + +void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); + if (pVnode != NULL) { + dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg); + dndReleaseVnode(pDnode, pVnode); + } +} + +void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); + if (pVnode != NULL) { + dndWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg); + dndReleaseVnode(pDnode, pVnode); + } +} + +void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); + if (pVnode != NULL) { + dndWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg); + dndReleaseVnode(pDnode, pVnode); + } +} + +void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); + if (pVnode != NULL) { + dndWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg); + dndReleaseVnode(pDnode, pVnode); + } +} + +static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg) { + SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + if (pVnode == NULL) { + return -1; + } + + int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg); + dndReleaseVnode(pDnode, pVnode); + return code; +} + +static int32_t dndInitVnodeMgmtWorker(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + SWorkerPool *pPool = &pMgmt->mgmtPool; + pPool->name = "vnode-mgmt"; + pPool->min = 1; + pPool->max = 1; + if (tWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + pMgmt->pMgmtQ = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)dndProcessVnodeMgmtQueue); + if (pMgmt->pMgmtQ == NULL) { + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndCleanupVnodeMgmtWorker(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tWorkerFreeQueue(&pMgmt->mgmtPool, pMgmt->pMgmtQ); + tWorkerCleanup(&pMgmt->mgmtPool); + pMgmt->pMgmtQ = NULL; +} + +static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue); + if (pVnode->pQueryQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); + pVnode->pQueryQ = NULL; +} + +static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue); + if (pVnode->pFetchQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + pVnode->pFetchQ = NULL; +} + +static int32_t dndInitVnodeReadWorker(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + + int32_t maxFetchThreads = 4; + float threadsForQuery = MAX(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores, 1); + + SWorkerPool *pPool = &pMgmt->queryPool; + pPool->name = "vnode-query"; + pPool->min = (int32_t)threadsForQuery; + pPool->max = pPool->min; + if (tWorkerInit(pPool) != 0) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + pPool = &pMgmt->fetchPool; + pPool->name = "vnode-fetch"; + pPool->min = MIN(maxFetchThreads, pDnode->opt.numOfCores); + pPool->max = pPool->min; + if (tWorkerInit(pPool) != 0) { + TSDB_CODE_VND_OUT_OF_MEMORY; + } + + return 0; +} + +static void dndCleanupVnodeReadWorker(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tWorkerCleanup(&pMgmt->fetchPool); + tWorkerCleanup(&pMgmt->queryPool); +} + +static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue); + if (pVnode->pWriteQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); + pVnode->pWriteQ = NULL; +} + +static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue); + if (pVnode->pApplyQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + pVnode->pApplyQ = NULL; +} + +static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + SMWorkerPool *pPool = &pMgmt->writePool; + pPool->name = "vnode-write"; + pPool->max = tsNumOfCores; + if (tMWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndCleanupVnodeWriteWorker(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tMWorkerCleanup(&pMgmt->writePool); +} + +static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); + if (pVnode->pSyncQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pSyncQ); + pVnode->pSyncQ = NULL; +} + +static int32_t dndInitVnodeSyncWorker(SDnode *pDnode) { + int32_t maxThreads = tsNumOfCores / 2; + if (maxThreads < 1) maxThreads = 1; + + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + SMWorkerPool *pPool = &pMgmt->writePool; + pPool->name = "vnode-sync"; + pPool->max = maxThreads; + if (tMWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndCleanupVnodeSyncWorker(SDnode *pDnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tMWorkerCleanup(&pMgmt->syncPool); +} + +int32_t dndInitVnodes(SDnode *pDnode) { + dInfo("dnode-vnodes start to init"); + + if (dndInitVnodeReadWorker(pDnode) != 0) { + dError("failed to init vnodes read worker since %s", terrstr()); + return -1; + } + + if (dndInitVnodeWriteWorker(pDnode) != 0) { + dError("failed to init vnodes write worker since %s", terrstr()); + return -1; + } + + if (dndInitVnodeSyncWorker(pDnode) != 0) { + dError("failed to init vnodes sync worker since %s", terrstr()); + return -1; + } + + if (dndInitVnodeMgmtWorker(pDnode) != 0) { + dError("failed to init vnodes mgmt worker since %s", terrstr()); + return -1; + } + + if (dndOpenVnodes(pDnode) != 0) { + dError("failed to open vnodes since %s", terrstr()); + return -1; + } + + dInfo("dnode-vnodes is initialized"); + return 0; +} + +void dndCleanupVnodes(SDnode *pDnode) { + dInfo("dnode-vnodes start to clean up"); + dndCloseVnodes(pDnode); + dndCleanupVnodeReadWorker(pDnode); + dndCleanupVnodeWriteWorker(pDnode); + dndCleanupVnodeSyncWorker(pDnode); + dndCleanupVnodeMgmtWorker(pDnode); + dInfo("dnode-vnodes is cleaned up"); +} + +void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + + taosRLockLatch(&pMgmt->latch); + pLoads->num = taosHashGetSize(pMgmt->hash); + + int32_t v = 0; + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + if (ppVnode == NULL || *ppVnode == NULL) continue; + + SVnodeObj *pVnode = *ppVnode; + SVnodeLoad *pLoad = &pLoads->data[v++]; + + vnodeGetLoad(pVnode->pImpl, pLoad); + pLoad->vgId = htonl(pLoad->vgId); + pLoad->totalStorage = htobe64(pLoad->totalStorage); + pLoad->compStorage = htobe64(pLoad->compStorage); + pLoad->pointsWritten = htobe64(pLoad->pointsWritten); + pLoad->tablesNum = htobe64(pLoad->tablesNum); + + pIter = taosHashIterate(pMgmt->hash, pIter); + } + + taosRUnLockLatch(&pMgmt->latch); +} diff --git a/source/dnode/mgmt/impl/src/dnodeMnode.c b/source/dnode/mgmt/impl/src/dnodeMnode.c deleted file mode 100644 index 06b28aeea9..0000000000 --- a/source/dnode/mgmt/impl/src/dnodeMnode.c +++ /dev/null @@ -1,608 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dnodeMnode.h" -#include "cJSON.h" -#include "dnodeDnode.h" -#include "dnodeTransport.h" -#include "mnode.h" -#include "tlockfree.h" -#include "tqueue.h" -#include "tstep.h" -#include "tworker.h" - -static struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SWorkerPool mgmtPool; - SWorkerPool readPool; - SWorkerPool writePool; - SWorkerPool syncPool; - taos_queue pReadQ; - taos_queue pWriteQ; - taos_queue pApplyQ; - taos_queue pSyncQ; - taos_queue pMgmtQ; - SSteps *pSteps; - SMnode *pMnode; - SRWLatch latch; -} tsMnode = {0}; - -static int32_t dnodeAllocMnodeReadQueue(); -static void dnodeFreeMnodeReadQueue(); -static int32_t dnodeAllocMnodeWriteQueue(); -static void dnodeFreeMnodeWriteQueue(); -static int32_t dnodeAllocMnodeApplyQueue(); -static void dnodeFreeMnodeApplyQueue(); -static int32_t dnodeAllocMnodeSyncQueue(); -static void dnodeFreeMnodeSyncQueue(); - -static SMnode *dnodeAcquireMnode() { - SMnode *pMnode = NULL; - taosRLockLatch(&tsMnode.latch); - - if (tsMnode.deployed) { - atomic_add_fetch_32(&tsMnode.refCount, 1); - pMnode = tsMnode.pMnode; - } - - taosRUnLockLatch(&tsMnode.latch); - return pMnode; -} - -static void dnodeReleaseMnode(SMnode *pMnode) { - taosRLockLatch(&tsMnode.latch); - atomic_sub_fetch_32(&tsMnode.refCount, 1); - taosRUnLockLatch(&tsMnode.latch); -} - -static int32_t dnodeReadMnodeFile() { - int32_t code = TSDB_CODE_DND_READ_MNODE_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 300; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - FILE *fp = NULL; - char file[PATH_MAX + 20] = {0}; - - snprintf(file, sizeof(file), "%s/mnode.json", tsDnodeDir); - fp = fopen(file, "r"); - if (!fp) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_MNODE_OVER; - } - - len = (int32_t)fread(content, 1, maxLen, fp); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_MNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_MNODE_OVER; - } - - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_String) { - dError("failed to read %s since deployed not found", file); - goto PRASE_MNODE_OVER; - } - tsMnode.deployed = atoi(deployed->valuestring); - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_String) { - dError("failed to read %s since dropped not found", file); - goto PRASE_MNODE_OVER; - } - tsMnode.dropped = atoi(dropped->valuestring); - - code = 0; - dInfo("succcessed to read file %s", file); - -PRASE_MNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); - - return code; -} - -static int32_t dnodeWriteMnodeFile() { - char file[PATH_MAX + 20] = {0}; - char realfile[PATH_MAX + 20] = {0}; - snprintf(file, sizeof(file), "%s/mnode.json.bak", tsDnodeDir); - snprintf(realfile, sizeof(realfile), "%s/mnode.json", tsDnodeDir); - - FILE *fp = fopen(file, "w"); - if (!fp) { - dError("failed to write %s since %s", file, strerror(errno)); - return TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR; - } - - int32_t len = 0; - int32_t maxLen = 300; - char *content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"deployed\": \"%d\",\n", tsMnode.deployed); - len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", tsMnode.dropped); - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); - free(content); - - int32_t code = taosRenameFile(file, realfile); - if (code != 0) { - dError("failed to rename %s since %s", file, tstrerror(code)); - return TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR; - } - - dInfo("successed to write %s", realfile); - return 0; -} - -static int32_t dnodeStartMnode() { - int32_t code = dnodeAllocMnodeReadQueue(); - if (code != 0) { - return code; - } - - code = dnodeAllocMnodeWriteQueue(); - if (code != 0) { - return code; - } - - code = dnodeAllocMnodeApplyQueue(); - if (code != 0) { - return code; - } - - code = dnodeAllocMnodeSyncQueue(); - if (code != 0) { - return code; - } - - taosWLockLatch(&tsMnode.latch); - tsMnode.deployed = 1; - taosWUnLockLatch(&tsMnode.latch); - - return mnodeStart(NULL); -} - -static void dnodeStopMnode() { - taosWLockLatch(&tsMnode.latch); - tsMnode.deployed = 0; - taosWUnLockLatch(&tsMnode.latch); - - dnodeReleaseMnode(); - - while (tsMnode.refCount > 0) taosMsleep(10); - while (!taosQueueEmpty(tsMnode.pReadQ)) taosMsleep(10); - while (!taosQueueEmpty(tsMnode.pApplyQ)) taosMsleep(10); - while (!taosQueueEmpty(tsMnode.pWriteQ)) taosMsleep(10); - while (!taosQueueEmpty(tsMnode.pSyncQ)) taosMsleep(10); - - dnodeFreeMnodeReadQueue(); - dnodeFreeMnodeWriteQueue(); - dnodeFreeMnodeApplyQueue(); - dnodeFreeMnodeSyncQueue(); -} - -static int32_t dnodeUnDeployMnode() { - tsMnode.dropped = 1; - int32_t code = dnodeWriteMnodeFile(); - if (code != 0) { - tsMnode.dropped = 0; - dError("failed to undeploy mnode since %s", tstrerror(code)); - return code; - } - - dnodeStopMnode(); - mnodeUnDeploy(); - dnodeWriteMnodeFile(); - - return code; -} - -static int32_t dnodeDeployMnode(SMnodeCfg *pCfg) { - int32_t code = mnodeDeploy(pCfg); - if (code != 0) { - dError("failed to deploy mnode since %s", tstrerror(code)); - return code; - } - - code = dnodeStartMnode(); - if (code != 0) { - dnodeUnDeployMnode(); - dError("failed to deploy mnode since %s", tstrerror(code)); - return code; - } - - code = dnodeWriteMnodeFile(); - if (code != 0) { - dnodeUnDeployMnode(); - dError("failed to deploy mnode since %s", tstrerror(code)); - return code; - } - - dInfo("deploy mnode success"); - return code; -} - -static int32_t dnodeAlterMnode(SMnodeCfg *pCfg) { - int32_t code = dnodeAcquireMnode(); - if (code == 0) { - code = mnodeAlter(pCfg); - dnodeReleaseMnode(); - } - return code; -} - -static SCreateMnodeMsg *dnodeParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { - SCreateMnodeMsg *pMsg = pRpcMsg->pCont; - pMsg->dnodeId = htonl(pMsg->dnodeId); - for (int32_t i = 0; i < pMsg->replica; ++i) { - pMsg->replicas[i].port = htons(pMsg->replicas[i].port); - } - return pMsg; -} - -static int32_t dnodeProcessCreateMnodeReq(SRpcMsg *pRpcMsg) { - SAlterMnodeMsg *pMsg = (SAlterMnodeMsg *)dnodeParseCreateMnodeMsg(pRpcMsg->pCont); - - if (pMsg->dnodeId != dnodeGetDnodeId()) { - return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE; - } else { - SMnodeCfg cfg = {0}; - cfg.replica = pMsg->replica; - memcpy(cfg.replicas, pMsg->replicas, sizeof(SReplica) * sizeof(TSDB_MAX_REPLICA)); - return dnodeDeployMnode(&cfg); - } -} - -static int32_t dnodeProcessAlterMnodeReq(SRpcMsg *pRpcMsg) { - SAlterMnodeMsg *pMsg = (SAlterMnodeMsg *)dnodeParseCreateMnodeMsg(pRpcMsg->pCont); - if (pMsg->dnodeId != dnodeGetDnodeId()) { - return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE; - } else { - SMnodeCfg cfg = {0}; - cfg.replica = pMsg->replica; - memcpy(cfg.replicas, pMsg->replicas, sizeof(SReplica) * sizeof(TSDB_MAX_REPLICA)); - return dnodeAlterMnode(&cfg); - } -} - -static int32_t dnodeProcessDropMnodeReq(SRpcMsg *pMsg) { - SAlterMnodeMsg *pCfg = pMsg->pCont; - pCfg->dnodeId = htonl(pCfg->dnodeId); - - if (pCfg->dnodeId != dnodeGetDnodeId()) { - return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE; - } else { - return dnodeUnDeployMnode(); - } -} - -static void dnodeProcessMnodeMgmtQueue(void *unused, SRpcMsg *pMsg) { - int32_t code = 0; - - switch (pMsg->msgType) { - case TSDB_MSG_TYPE_CREATE_MNODE_IN: - code = dnodeProcessCreateMnodeReq(pMsg); - break; - case TSDB_MSG_TYPE_ALTER_MNODE_IN: - code = dnodeProcessAlterMnodeReq(pMsg); - break; - case TSDB_MSG_TYPE_DROP_MNODE_IN: - code = dnodeProcessDropMnodeReq(pMsg); - break; - default: - code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - break; - } - - SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; - rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dnodeProcessMnodeReadQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_READ); } - -static void dnodeProcessMnodeWriteQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_WRITE); } - -static void dnodeProcessMnodeApplyQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_APPLY); } - -static void dnodeProcessMnodeSyncQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_SYNC); } - -static int32_t dnodeWriteMnodeMsgToQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { - int32_t code = 0; - SMnodeMsg *pMsg = NULL; - - if (pQueue == NULL) { - code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - } else { - pMsg = mnodeInitMsg(pRpcMsg); - if (pMsg == NULL) { - code = terrno; - } - } - - if (code == 0) { - code = taosWriteQitem(pQueue, pMsg); - } - - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); - rpcFreeCont(pRpcMsg->pCont); - } -} - -void dnodeProcessMnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { dnodeWriteMnodeMsgToQueue(tsMnode.pMgmtQ, pMsg); } - -void dnodeProcessMnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { - if (dnodeAcquireMnode() == 0) { - dnodeWriteMnodeMsgToQueue(tsMnode.pWriteQ, pMsg); - dnodeReleaseMnode(); - } else { - dnodeSendRedirectMsg(NULL, pMsg, 0); - } -} - -void dnodeProcessMnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { - int32_t code = dnodeAcquireMnode(); - if (code == 0) { - dnodeWriteMnodeMsgToQueue(tsMnode.pSyncQ, pMsg); - dnodeReleaseMnode(); - } else { - SRpcMsg rsp = {.handle = pMsg->handle, .code = code}; - rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); - } -} - -void dnodeProcessMnodeReadMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { - if (dnodeAcquireMnode() == 0) { - dnodeWriteMnodeMsgToQueue(tsMnode.pReadQ, pMsg); - dnodeReleaseMnode(); - } else { - dnodeSendRedirectMsg(NULL, pMsg, 0); - } -} - -static int32_t dnodePutMsgIntoMnodeApplyQueue(SMnodeMsg *pMsg) { - int32_t code = dnodeAcquireMnode(); - if (code != 0) return code; - - code = taosWriteQitem(tsMnode.pApplyQ, pMsg); - dnodeReleaseMnode(); - return code; -} - -static int32_t dnodeAllocMnodeMgmtQueue() { - tsMnode.pMgmtQ = tWorkerAllocQueue(&tsMnode.mgmtPool, NULL, (FProcessItem)dnodeProcessMnodeMgmtQueue); - if (tsMnode.pMgmtQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeMnodeMgmtQueue() { - tWorkerFreeQueue(&tsMnode.mgmtPool, tsMnode.pMgmtQ); - tsMnode.pMgmtQ = NULL; -} - -static int32_t dnodeInitMnodeMgmtWorker() { - SWorkerPool *pPool = &tsMnode.mgmtPool; - pPool->name = "mnode-mgmt"; - pPool->min = 1; - pPool->max = 1; - return tWorkerInit(pPool); -} - -static void dnodeCleanupMnodeMgmtWorker() { tWorkerCleanup(&tsMnode.mgmtPool); } - -static int32_t dnodeAllocMnodeReadQueue() { - tsMnode.pReadQ = tWorkerAllocQueue(&tsMnode.readPool, NULL, (FProcessItem)dnodeProcessMnodeReadQueue); - if (tsMnode.pReadQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeMnodeReadQueue() { - tWorkerFreeQueue(&tsMnode.readPool, tsMnode.pReadQ); - tsMnode.pReadQ = NULL; -} - -static int32_t dnodeInitMnodeReadWorker() { - SWorkerPool *pPool = &tsMnode.readPool; - pPool->name = "mnode-read"; - pPool->min = 0; - pPool->max = 1; - return tWorkerInit(pPool); -} - -static void dnodeCleanupMnodeReadWorker() { tWorkerCleanup(&tsMnode.readPool); } - -static int32_t dnodeAllocMnodeWriteQueue() { - tsMnode.pWriteQ = tWorkerAllocQueue(&tsMnode.writePool, NULL, (FProcessItem)dnodeProcessMnodeWriteQueue); - if (tsMnode.pWriteQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeMnodeWriteQueue() { - tWorkerFreeQueue(&tsMnode.writePool, tsMnode.pWriteQ); - tsMnode.pWriteQ = NULL; -} - -static int32_t dnodeAllocMnodeApplyQueue() { - tsMnode.pApplyQ = tWorkerAllocQueue(&tsMnode.writePool, NULL, (FProcessItem)dnodeProcessMnodeApplyQueue); - if (tsMnode.pApplyQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeMnodeApplyQueue() { - tWorkerFreeQueue(&tsMnode.writePool, tsMnode.pApplyQ); - tsMnode.pApplyQ = NULL; -} - -static int32_t dnodeInitMnodeWriteWorker() { - SWorkerPool *pPool = &tsMnode.writePool; - pPool->name = "mnode-write"; - pPool->min = 0; - pPool->max = 1; - return tWorkerInit(pPool); -} - -static void dnodeCleanupMnodeWriteWorker() { tWorkerCleanup(&tsMnode.writePool); } - -static int32_t dnodeAllocMnodeSyncQueue() { - tsMnode.pSyncQ = tWorkerAllocQueue(&tsMnode.syncPool, NULL, (FProcessItem)dnodeProcessMnodeSyncQueue); - if (tsMnode.pSyncQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeMnodeSyncQueue() { - tWorkerFreeQueue(&tsMnode.syncPool, tsMnode.pSyncQ); - tsMnode.pSyncQ = NULL; -} - -static int32_t dnodeInitMnodeSyncWorker() { - SWorkerPool *pPool = &tsMnode.syncPool; - pPool->name = "mnode-sync"; - pPool->min = 0; - pPool->max = 1; - return tWorkerInit(pPool); -} - -static void dnodeCleanupMnodeSyncWorker() { tWorkerCleanup(&tsMnode.syncPool); } - -static int32_t dnodeInitMnodeModule() { - taosInitRWLatch(&tsMnode.latch); - - SMnodeOptions para; - para.dnodeId = dnodeGetDnodeId(); - para.clusterId = dnodeGetClusterId(); - para.sendMsgToDnodeFp = dnodeSendMsgToDnode; - para.sendMsgToMnodeFp = dnodeSendMsgToMnode; - para.sendRedirectMsgFp = dnodeSendRedirectMsg; - - tsMnode.pMnode = mnodeCreate(para); - if (tsMnode.pMnode != NULL) { - return -1; - } - return 0; -} - -static void dnodeCleanupMnodeModule() { mnodeDrop(NULL); } - -static bool dnodeNeedDeployMnode() { - if (dnodeGetDnodeId() > 0) return false; - if (dnodeGetClusterId() > 0) return false; - if (strcmp(tsFirst, tsLocalEp) != 0) return false; - return true; -} - -static int32_t dnodeOpenMnode() { - int32_t code = dnodeReadMnodeFile(); - if (code != 0) { - dError("failed to read open mnode since %s", tstrerror(code)); - return code; - } - - if (tsMnode.dropped) { - dInfo("mnode already dropped, undeploy it"); - return dnodeUnDeployMnode(); - } - - if (!tsMnode.deployed) { - bool needDeploy = dnodeNeedDeployMnode(); - if (!needDeploy) return 0; - - dInfo("start to deploy mnode"); - SMnodeCfg cfg = {.replica = 1}; - cfg.replicas[0].port = tsServerPort; - tstrncpy(cfg.replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); - code = dnodeDeployMnode(&cfg); - } else { - dInfo("start to open mnode"); - return dnodeStartMnode(); - } -} - -static void dnodeCloseMnode() { - if (dnodeAcquireMnode() == 0) { - dnodeStopMnode(); - } -} - -int32_t dnodeInitMnode() { - dInfo("dnode-mnode start to init"); - - SSteps *pSteps = taosStepInit(6, dnodeReportStartup); - taosStepAdd(pSteps, "dnode-mnode-env", dnodeInitMnodeModule, dnodeCleanupMnodeModule); - taosStepAdd(pSteps, "dnode-mnode-mgmt", dnodeInitMnodeMgmtWorker, dnodeCleanupMnodeMgmtWorker); - taosStepAdd(pSteps, "dnode-mnode-read", dnodeInitMnodeReadWorker, dnodeCleanupMnodeReadWorker); - taosStepAdd(pSteps, "dnode-mnode-write", dnodeInitMnodeWriteWorker, dnodeCleanupMnodeWriteWorker); - taosStepAdd(pSteps, "dnode-mnode-sync", dnodeInitMnodeSyncWorker, dnodeCleanupMnodeSyncWorker); - taosStepAdd(pSteps, "dnode-mnode", dnodeOpenMnode, dnodeCloseMnode); - - tsMnode.pSteps = pSteps; - int32_t code = taosStepExec(pSteps); - - if (code != 0) { - dError("dnode-mnode init failed since %s", tstrerror(code)); - } else { - dInfo("dnode-mnode is initialized"); - } -} - -void dnodeCleanupMnode() { - if (tsMnode.pSteps == NULL) { - dInfo("dnode-mnode start to clean up"); - taosStepCleanup(tsMnode.pSteps); - tsMnode.pSteps = NULL; - dInfo("dnode-mnode is cleaned up"); - } -} - -int32_t dnodeGetUserAuthFromMnode(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SMnode *pMnode = dnodeAcquireMnode(); - if (pMnode == NULL) { - dTrace("failed to get user auth since mnode not deployed"); - terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; - return -1; - } - - int32_t code = mnodeRetriveAuth(pMnode, user, spi, encrypt, secret, ckey); - dnodeReleaseMnode(pMnode); - return code; -} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dnodeTransport.c b/source/dnode/mgmt/impl/src/dnodeTransport.c deleted file mode 100644 index c1e8955625..0000000000 --- a/source/dnode/mgmt/impl/src/dnodeTransport.c +++ /dev/null @@ -1,395 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/* this file is mainly responsible for the communication between DNODEs. Each - * dnode works as both server and client. Dnode may send status, grant, config - * messages to mnode, mnode may send create/alter/drop table/vnode messages - * to dnode. All theses messages are handled from here - */ - -#define _DEFAULT_SOURCE -#include "dnodeTransport.h" -#include "dnodeDnode.h" -#include "dnodeMnode.h" -#include "dnodeVnodes.h" - -static struct { - void *peerRpc; - void *shellRpc; - void *clientRpc; - MsgFp msgFp[TSDB_MSG_TYPE_MAX]; -} tsTrans; - -static void dnodeInitMsgFp() { - // msg from client to dnode - tsTrans.msgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_QUERY] = dnodeProcessVnodeQueryMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_FETCH] = dnodeProcessVnodeFetchMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_TABLE] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_TABLE_META] = dnodeProcessVnodeQueryMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_TABLES_META] = dnodeProcessVnodeQueryMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dnodeProcessVnodeQueryMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dnodeProcessVnodeQueryMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_MQ_ACK] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_MQ_RESET] = dnodeProcessVnodeWriteMsg; - - // msg from client to mnode - tsTrans.msgFp[TSDB_MSG_TYPE_CONNECT] = dnodeProcessMnodeReadMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_ACCT] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_ACCT] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_ACCT] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_USER] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_USER] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_USER] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_DNODE] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_DNODE] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_DB] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_DB] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_USE_DB] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_DB] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_DB] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_TOPIC] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_FUNCTION] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dnodeProcessMnodeReadMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_KILL_CONN] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dnodeProcessMnodeReadMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_SHOW] = dnodeProcessMnodeReadMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dnodeProcessMnodeReadMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC] = dnodeProcessMnodeReadMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dnodeProcessMnodeWriteMsg; - - // message from client to dnode - tsTrans.msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeProcessDnodeMsg; - - // message from mnode to vnode - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dnodeProcessVnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dnodeProcessMnodeWriteMsg; - - // message from mnode to dnode - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dnodeProcessVnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN] = dnodeProcessVnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN] = dnodeProcessVnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN] = dnodeProcessVnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN] = dnodeProcessVnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN] = dnodeProcessVnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN] = dnodeProcessMnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN] = dnodeProcessMnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN] = dnodeProcessMnodeMgmtMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN] = dnodeProcessDnodeMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP] = dnodeProcessMnodeWriteMsg; - - // message from dnode to mnode - tsTrans.msgFp[TSDB_MSG_TYPE_AUTH] = dnodeProcessMnodeReadMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_RSP] = dnodeProcessDnodeMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_GRANT] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_GRANT_RSP] = dnodeProcessDnodeMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_STATUS] = dnodeProcessMnodeWriteMsg; - tsTrans.msgFp[TSDB_MSG_TYPE_STATUS_RSP] = dnodeProcessDnodeMsg; -} - -static void dnodeProcessPeerReq(SRpcMsg *pMsg, SEpSet *pEpSet) { - SRpcMsg rspMsg = {.handle = pMsg->handle}; - int32_t msgType = pMsg->msgType; - - if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) { - dnodeProcessDnodeMsg(pMsg, pEpSet); - return; - } - - if (dnodeGetStat() != DN_STAT_RUNNING) { - rspMsg.code = TSDB_CODE_APP_NOT_READY; - rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); - dTrace("RPC %p, peer req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]); - return; - } - - if (pMsg->pCont == NULL) { - rspMsg.code = TSDB_CODE_DND_INVALID_MSG_LEN; - rpcSendResponse(&rspMsg); - return; - } - - MsgFp fp = tsTrans.msgFp[msgType]; - if (fp != NULL) { - dTrace("RPC %p, peer req:%s will be processed", pMsg->handle, taosMsg[msgType]); - (*fp)(pMsg, pEpSet); - } else { - dError("RPC %p, peer req:%s not processed", pMsg->handle, taosMsg[msgType]); - rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); - } -} - -static int32_t dnodeInitPeerServer() { - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = tsDnodeDnodePort; - rpcInit.label = "DND-S"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dnodeProcessPeerReq; - rpcInit.sessions = TSDB_MAX_VNODES << 4; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - - tsTrans.peerRpc = rpcOpen(&rpcInit); - if (tsTrans.peerRpc == NULL) { - dError("failed to init peer rpc server"); - return -1; - } - - dInfo("dnode peer rpc server is initialized"); - return 0; -} - -static void dnodeCleanupPeerServer() { - if (tsTrans.peerRpc) { - rpcClose(tsTrans.peerRpc); - tsTrans.peerRpc = NULL; - dInfo("dnode peer server is closed"); - } -} - -static void dnodeProcessPeerRsp(SRpcMsg *pMsg, SEpSet *pEpSet) { - int32_t msgType = pMsg->msgType; - - if (dnodeGetStat() == DN_STAT_STOPPED) { - if (pMsg == NULL || pMsg->pCont == NULL) return; - dTrace("RPC %p, peer rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]); - rpcFreeCont(pMsg->pCont); - return; - } - - MsgFp fp = tsTrans.msgFp[msgType]; - if (fp != NULL) { - dTrace("RPC %p, peer rsp:%s will be processed, code:%s", pMsg->handle, taosMsg[msgType], tstrerror(pMsg->code)); - (*fp)(pMsg, pEpSet); - } else { - dDebug("RPC %p, peer rsp:%s not processed", pMsg->handle, taosMsg[msgType]); - } - - rpcFreeCont(pMsg->pCont); -} - -static int32_t dnodeInitClient() { - char secret[TSDB_KEY_LEN] = "secret"; - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "DND-C"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dnodeProcessPeerRsp; - rpcInit.sessions = TSDB_MAX_VNODES << 4; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = "t"; - rpcInit.ckey = "key"; - rpcInit.secret = secret; - - tsTrans.clientRpc = rpcOpen(&rpcInit); - if (tsTrans.clientRpc == NULL) { - dError("failed to init peer rpc client"); - return -1; - } - - dInfo("dnode peer rpc client is initialized"); - return 0; -} - -static void dnodeCleanupClient() { - if (tsTrans.clientRpc) { - rpcClose(tsTrans.clientRpc); - tsTrans.clientRpc = NULL; - dInfo("dnode peer rpc client is closed"); - } -} - -static void dnodeProcessShellReq(SRpcMsg *pMsg, SEpSet *pEpSet) { - SRpcMsg rspMsg = {.handle = pMsg->handle}; - int32_t msgType = pMsg->msgType; - - if (dnodeGetStat() == DN_STAT_STOPPED) { - dError("RPC %p, shell req:%s is ignored since dnode exiting", pMsg->handle, taosMsg[msgType]); - rspMsg.code = TSDB_CODE_DND_EXITING; - rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); - return; - } else if (dnodeGetStat() != DN_STAT_RUNNING) { - dError("RPC %p, shell req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]); - rspMsg.code = TSDB_CODE_APP_NOT_READY; - rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); - return; - } - - if (pMsg->pCont == NULL) { - rspMsg.code = TSDB_CODE_DND_INVALID_MSG_LEN; - rpcSendResponse(&rspMsg); - return; - } - - MsgFp fp = tsTrans.msgFp[msgType]; - if (fp != NULL) { - dTrace("RPC %p, shell req:%s will be processed", pMsg->handle, taosMsg[msgType]); - (*fp)(pMsg, pEpSet); - } else { - dError("RPC %p, shell req:%s is not processed", pMsg->handle, taosMsg[msgType]); - rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); - } -} - -static void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { - SEpSet epSet = {0}; - dnodeGetMnodeEpSetForPeer(&epSet); - rpcSendRecv(tsTrans.clientRpc, &epSet, rpcMsg, rpcRsp); -} - -static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - int32_t code = dnodeGetUserAuthFromMnode(user, spi, encrypt, secret, ckey); - if (code != TSDB_CODE_APP_NOT_READY) return code; - - SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); - tstrncpy(pMsg->user, user, sizeof(pMsg->user)); - - dDebug("user:%s, send auth msg to mnodes", user); - SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TSDB_MSG_TYPE_AUTH}; - SRpcMsg rpcRsp = {0}; - dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code)); - } else { - dDebug("user:%s, auth msg received from mnodes", user); - SAuthRsp *pRsp = rpcRsp.pCont; - memcpy(secret, pRsp->secret, TSDB_KEY_LEN); - memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); - *spi = pRsp->spi; - *encrypt = pRsp->encrypt; - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - -static int32_t dnodeInitShellServer() { - dnodeInitMsgFp(); - - int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); - if (numOfThreads < 1) { - numOfThreads = 1; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = tsDnodeShellPort; - rpcInit.label = "SHELL"; - rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dnodeProcessShellReq; - rpcInit.sessions = tsMaxShellConns; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dnodeRetrieveUserAuthInfo; - - tsTrans.shellRpc = rpcOpen(&rpcInit); - if (tsTrans.shellRpc == NULL) { - dError("failed to init shell rpc server"); - return -1; - } - -#if 1 - SRpcMsg rpcMsg = {0}; - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - SCreateUserMsg *pMsg = (SCreateUserMsg*)rpcMsg.pCont; - strcpy(pMsg->user, "u1"); - strcpy(pMsg->pass, "up1"); - dnodeProcessShellReq(&rpcMsg, NULL); - -#endif - dInfo("dnode shell rpc server is initialized"); - return 0; -} - -static void dnodeCleanupShellServer() { - if (tsTrans.shellRpc) { - rpcClose(tsTrans.shellRpc); - tsTrans.shellRpc = NULL; - } -} - -int32_t dnodeInitTrans() { - if (dnodeInitClient() != 0) { - return -1; - } - - if (dnodeInitPeerServer() != 0) { - return -1; - } - - if (dnodeInitShellServer() != 0) { - return -1; - } - - return 0; -} - -void dnodeCleanupTrans() { - dnodeCleanupShellServer(); - dnodeCleanupPeerServer(); - dnodeCleanupClient(); -} - -void dnodeSendMsgToDnode(SDnode *pDnode, SEpSet *epSet, SRpcMsg *rpcMsg) { - #if 0 - rpcSendRequest(tsTrans.clientRpc, epSet, rpcMsg, NULL); - #endif - } - -void dnodeSendMsgToMnode(SDnode *pDnode, SRpcMsg *rpcMsg) { - SEpSet epSet = {0}; - dnodeGetMnodeEpSetForPeer(&epSet); - dnodeSendMsgToDnode(NULL, &epSet, rpcMsg); -} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dnodeVnodes.c b/source/dnode/mgmt/impl/src/dnodeVnodes.c deleted file mode 100644 index 7eaa82ba93..0000000000 --- a/source/dnode/mgmt/impl/src/dnodeVnodes.c +++ /dev/null @@ -1,1029 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dnodeVnodes.h" -#include "dnodeTransport.h" -#include "cJSON.h" -#include "thash.h" -#include "tlockfree.h" -#include "tqueue.h" -#include "tstep.h" -#include "tthread.h" -#include "tworker.h" -#include "vnode.h" - -typedef struct { - int32_t vgId; - int32_t refCount; - int8_t dropped; - int8_t accessState; - SVnode *pImpl; - taos_queue pWriteQ; - taos_queue pSyncQ; - taos_queue pApplyQ; - taos_queue pQueryQ; - taos_queue pFetchQ; -} SVnodeObj; - -typedef struct { - pthread_t *threadId; - int32_t threadIndex; - int32_t failed; - int32_t opened; - int32_t vnodeNum; - SVnodeObj *pVnodes; -} SVThread; - -static struct { - SHashObj *hash; - SWorkerPool mgmtPool; - SWorkerPool queryPool; - SWorkerPool fetchPool; - SMWorkerPool syncPool; - SMWorkerPool writePool; - taos_queue pMgmtQ; - SSteps *pSteps; - int32_t openVnodes; - int32_t totalVnodes; - SRWLatch latch; -} tsVnodes; - -static int32_t dnodeAllocVnodeQueryQueue(SVnodeObj *pVnode); -static void dnodeFreeVnodeQueryQueue(SVnodeObj *pVnode); -static int32_t dnodeAllocVnodeFetchQueue(SVnodeObj *pVnode); -static void dnodeFreeVnodeFetchQueue(SVnodeObj *pVnode); -static int32_t dnodeAllocVnodeWriteQueue(SVnodeObj *pVnode); -static void dnodeFreeVnodeWriteQueue(SVnodeObj *pVnode); -static int32_t dnodeAllocVnodeApplyQueue(SVnodeObj *pVnode); -static void dnodeFreeVnodeApplyQueue(SVnodeObj *pVnode); -static int32_t dnodeAllocVnodeSyncQueue(SVnodeObj *pVnode); -static void dnodeFreeVnodeSyncQueue(SVnodeObj *pVnode); - -static SVnodeObj *dnodeAcquireVnode(int32_t vgId) { - SVnodeObj *pVnode = NULL; - int32_t refCount = 0; - - taosRLockLatch(&tsVnodes.latch); - taosHashGetClone(tsVnodes.hash, &vgId, sizeof(int32_t), (void *)&pVnode); - if (pVnode == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - } else { - refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - } - taosRUnLockLatch(&tsVnodes.latch); - - dTrace("vgId:%d, accquire vnode, refCount:%d", pVnode->vgId, refCount); - return pVnode; -} - -static void dnodeReleaseVnode(SVnodeObj *pVnode) { - int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); -} - -static int32_t dnodeCreateVnodeWrapper(int32_t vgId, SVnode *pImpl) { - SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); - if (pVnode == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - - pVnode->vgId = vgId; - pVnode->refCount = 0; - pVnode->dropped = 0; - pVnode->accessState = TSDB_VN_ALL_ACCCESS; - pVnode->pImpl = pImpl; - - int32_t code = dnodeAllocVnodeQueryQueue(pVnode); - if (code != 0) { - return code; - } - - code = dnodeAllocVnodeFetchQueue(pVnode); - if (code != 0) { - return code; - } - - code = dnodeAllocVnodeWriteQueue(pVnode); - if (code != 0) { - return code; - } - - code = dnodeAllocVnodeApplyQueue(pVnode); - if (code != 0) { - return code; - } - - code = dnodeAllocVnodeSyncQueue(pVnode); - if (code != 0) { - return code; - } - - taosWLockLatch(&tsVnodes.latch); - code = taosHashPut(tsVnodes.hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); - taosWUnLockLatch(&tsVnodes.latch); - - return code; -} - -static void dnodeDropVnodeWrapper(SVnodeObj *pVnode) { - taosWLockLatch(&tsVnodes.latch); - taosHashRemove(tsVnodes.hash, &pVnode->vgId, sizeof(int32_t)); - taosWUnLockLatch(&tsVnodes.latch); - - // wait all queue empty - dnodeReleaseVnode(pVnode); - while (pVnode->refCount > 0) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); - - dnodeFreeVnodeQueryQueue(pVnode); - dnodeFreeVnodeFetchQueue(pVnode); - dnodeFreeVnodeWriteQueue(pVnode); - dnodeFreeVnodeApplyQueue(pVnode); - dnodeFreeVnodeSyncQueue(pVnode); -} - -static SVnodeObj **dnodeGetVnodesFromHash(int32_t *numOfVnodes) { - taosRLockLatch(&tsVnodes.latch); - - int32_t num = 0; - int32_t size = taosHashGetSize(tsVnodes.hash); - SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); - - void *pIter = taosHashIterate(tsVnodes.hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - SVnodeObj *pVnode = *ppVnode; - if (pVnode) { - num++; - if (num < size) { - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, accquire vnode, refCount:%d", pVnode->vgId, refCount); - pVnodes[num] = (*ppVnode); - } - } - pIter = taosHashIterate(tsVnodes.hash, pIter); - } - - taosRUnLockLatch(&tsVnodes.latch); - *numOfVnodes = num; - - return pVnodes; -} - -static int32_t dnodeGetVnodesFromFile(SVnodeObj **ppVnodes, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 30000; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - FILE *fp = NULL; - char file[PATH_MAX + 20] = {0}; - SVnodeObj *pVnodes = NULL; - - snprintf(file, PATH_MAX + 20, "%s/vnodes.json", tsVnodeDir); - - fp = fopen(file, "r"); - if (!fp) { - dDebug("file %s not exist", file); - code = 0; - goto PRASE_VNODE_OVER; - } - - len = (int32_t)fread(content, 1, maxLen, fp); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto PRASE_VNODE_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto PRASE_VNODE_OVER; - } - - cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); - if (!vnodes || vnodes->type != cJSON_Array) { - dError("failed to read %s since vnodes not found", file); - goto PRASE_VNODE_OVER; - } - - int32_t vnodesNum = cJSON_GetArraySize(vnodes); - if (vnodesNum <= 0) { - dError("failed to read %s since vnodes size:%d invalid", file, vnodesNum); - goto PRASE_VNODE_OVER; - } - - pVnodes = calloc(vnodesNum, sizeof(SVnodeObj)); - if (pVnodes == NULL) { - dError("failed to read %s since out of memory", file); - goto PRASE_VNODE_OVER; - } - - for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); - SVnodeObj *pVnode = &pVnodes[i]; - - cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_String) { - dError("failed to read %s since vgId not found", file); - goto PRASE_VNODE_OVER; - } - pVnode->vgId = atoi(vgId->valuestring); - - cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_String) { - dError("failed to read %s since dropped not found", file); - goto PRASE_VNODE_OVER; - } - pVnode->dropped = atoi(vnode->valuestring); - } - - code = 0; - dInfo("succcessed to read file %s", file); - -PRASE_VNODE_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); - - return code; -} - -static int32_t dnodeWriteVnodesToFile() { - char file[PATH_MAX + 20] = {0}; - char realfile[PATH_MAX + 20] = {0}; - snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", tsVnodeDir); - snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", tsVnodeDir); - - FILE *fp = fopen(file, "w"); - if (!fp) { - dError("failed to write %s since %s", file, strerror(errno)); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 30000; - char *content = calloc(1, maxLen + 1); - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = dnodeGetVnodesFromHash(&numOfVnodes); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n"); - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId); - len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped); - if (i < numOfVnodes - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); - free(content); - terrno = 0; - - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - dnodeReleaseVnode(pVnode); - } - - if (pVnodes != NULL) { - free(pVnodes); - } - - dInfo("successed to write %s", file); - return taosRenameFile(file, realfile); -} - -static int32_t dnodeCreateVnode(int32_t vgId, SVnodeCfg *pCfg) { - int32_t code = 0; - - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path),"%s/vnode%d", tsVnodeDir, vgId); - SVnode *pImpl = vnodeCreate(vgId, path, pCfg); - - if (pImpl == NULL) { - code = terrno; - return code; - } - - code = dnodeCreateVnodeWrapper(vgId, pImpl); - if (code != 0) { - vnodeDrop(pImpl); - return code; - } - - code = dnodeWriteVnodesToFile(); - if (code != 0) { - vnodeDrop(pImpl); - return code; - } - - return code; -} - -static int32_t dnodeDropVnode(SVnodeObj *pVnode) { - pVnode->dropped = 1; - - int32_t code = dnodeWriteVnodesToFile(); - if (code != 0) { - pVnode->dropped = 0; - return code; - } - - dnodeDropVnodeWrapper(pVnode); - vnodeDrop(pVnode->pImpl); - dnodeWriteVnodesToFile(); - return 0; -} - -static void *dnodeOpenVnodeFunc(void *param) { - SVThread *pThread = param; - - dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); - setThreadName("open-vnodes"); - - for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - SVnodeObj *pVnode = &pThread->pVnodes[v]; - - char stepDesc[TSDB_STEP_DESC_LEN] = {0}; - snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId, - tsVnodes.openVnodes, tsVnodes.totalVnodes); - dnodeReportStartup("open-vnodes", stepDesc); - - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path),"%s/vnode%d", tsVnodeDir, pVnode->vgId); - SVnode *pImpl = vnodeOpen(path, NULL); - if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex); - pThread->failed++; - } else { - dnodeCreateVnodeWrapper(pVnode->vgId, pImpl); - dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex); - pThread->opened++; - } - - atomic_add_fetch_32(&tsVnodes.openVnodes, 1); - } - - dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); - return NULL; -} - -static int32_t dnodeOpenVnodes() { - taosInitRWLatch(&tsVnodes.latch); - - tsVnodes.hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (tsVnodes.hash == NULL) { - dError("failed to init vnode hash"); - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - SVnodeObj *pVnodes = NULL; - int32_t numOfVnodes = 0; - int32_t code = dnodeGetVnodesFromFile(&pVnodes, &numOfVnodes); - if (code != TSDB_CODE_SUCCESS) { - dInfo("failed to get vnode list from disk since %s", tstrerror(code)); - return code; - } - - tsVnodes.totalVnodes = numOfVnodes; - - int32_t threadNum = tsNumOfCores; - int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - - SVThread *threads = calloc(threadNum, sizeof(SVThread)); - for (int32_t t = 0; t < threadNum; ++t) { - threads[t].threadIndex = t; - threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj)); - } - - for (int32_t v = 0; v < numOfVnodes; ++v) { - int32_t t = v % threadNum; - SVThread *pThread = &threads[t]; - pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v]; - } - - dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); - - for (int32_t t = 0; t < threadNum; ++t) { - SVThread *pThread = &threads[t]; - if (pThread->vnodeNum == 0) continue; - - pThread->threadId = taosCreateThread(dnodeOpenVnodeFunc, pThread); - if (pThread->threadId == NULL) { - dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); - } - } - - for (int32_t t = 0; t < threadNum; ++t) { - SVThread *pThread = &threads[t]; - taosDestoryThread(pThread->threadId); - pThread->threadId = NULL; - free(pThread->pVnodes); - } - free(threads); - - if (tsVnodes.openVnodes != tsVnodes.totalVnodes) { - dError("there are total vnodes:%d, opened:%d", tsVnodes.totalVnodes, tsVnodes.openVnodes); - return -1; - } else { - dInfo("total vnodes:%d open successfully", tsVnodes.totalVnodes); - } - - return TSDB_CODE_SUCCESS; -} - -static void dnodeCloseVnodes() { - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = dnodeGetVnodesFromHash(&numOfVnodes); - - for (int32_t i = 0; i < numOfVnodes; ++i) { - dnodeDropVnodeWrapper(pVnodes[i]); - } - if (pVnodes != NULL) { - free(pVnodes); - } - - if (tsVnodes.hash != NULL) { - taosHashCleanup(tsVnodes.hash); - tsVnodes.hash = NULL; - } - - dInfo("total vnodes:%d are all closed", numOfVnodes); -} - -static int32_t dnodeParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) { - SCreateVnodeMsg *pCreate = rpcMsg->pCont; - *vgId = htonl(pCreate->vgId); - - tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN); - pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize); - pCfg->totalBlocks = htonl(pCreate->totalBlocks); - pCfg->daysPerFile = htonl(pCreate->daysPerFile); - pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0); - pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1); - pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); - pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); - pCfg->precision = pCreate->precision; - pCfg->compression = pCreate->compression; - pCfg->cacheLastRow = pCreate->cacheLastRow; - pCfg->update = pCreate->update; - pCfg->quorum = pCreate->quorum; - pCfg->replica = pCreate->replica; - pCfg->walLevel = pCreate->walLevel; - pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod); - - for (int32_t i = 0; i < pCfg->replica; ++i) { - pCfg->replicas[i].port = htons(pCreate->replicas[i].port); - tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); - } - - return 0; -} - -static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) { - SDropVnodeMsg *pDrop = rpcMsg->pCont; - pDrop->vgId = htonl(pDrop->vgId); - return pDrop; -} - -static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = rpcMsg->pCont; - pAuth->vgId = htonl(pAuth->vgId); - return pAuth; -} - -static int32_t vnodeProcessCreateVnodeReq(SRpcMsg *rpcMsg) { - SVnodeCfg vnodeCfg = {0}; - int32_t vgId = 0; - - dnodeParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); - dDebug("vgId:%d, create vnode req is received", vgId); - - SVnodeObj *pVnode = dnodeAcquireVnode(vgId); - if (pVnode != NULL) { - dDebug("vgId:%d, already exist, return success", vgId); - dnodeReleaseVnode(pVnode); - return 0; - } - - int32_t code = dnodeCreateVnode(vgId, &vnodeCfg); - if (code != 0) { - dError("vgId:%d, failed to create vnode since %s", vgId, tstrerror(code)); - } - - return code; -} - -static int32_t vnodeProcessAlterVnodeReq(SRpcMsg *rpcMsg) { - SVnodeCfg vnodeCfg = {0}; - int32_t vgId = 0; - int32_t code = 0; - - dnodeParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); - dDebug("vgId:%d, alter vnode req is received", vgId); - - SVnodeObj *pVnode = dnodeAcquireVnode(vgId); - if (pVnode == NULL) { - code = terrno; - dDebug("vgId:%d, failed to alter vnode since %s", vgId, tstrerror(code)); - return code; - } - - code = vnodeAlter(pVnode->pImpl, &vnodeCfg); - if (code != 0) { - dError("vgId:%d, failed to alter vnode since %s", vgId, tstrerror(code)); - } - - dnodeReleaseVnode(pVnode); - return code; -} - -static int32_t vnodeProcessDropVnodeReq(SRpcMsg *rpcMsg) { - SDropVnodeMsg *pDrop = vnodeParseDropVnodeReq(rpcMsg); - - int32_t code = 0; - int32_t vgId = pDrop->vgId; - dDebug("vgId:%d, drop vnode req is received", vgId); - - SVnodeObj *pVnode = dnodeAcquireVnode(vgId); - if (pVnode == NULL) { - code = terrno; - dDebug("vgId:%d, failed to drop since %s", vgId, tstrerror(code)); - return code; - } - - code = dnodeDropVnode(pVnode); - if (code != 0) { - dnodeReleaseVnode(pVnode); - dError("vgId:%d, failed to drop vnode since %s", vgId, tstrerror(code)); - } - - return code; -} - -static int32_t vnodeProcessAuthVnodeReq(SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg); - - int32_t code = 0; - int32_t vgId = pAuth->vgId; - dDebug("vgId:%d, auth vnode req is received", vgId); - - SVnodeObj *pVnode = dnodeAcquireVnode(vgId); - if (pVnode == NULL) { - code = terrno; - dDebug("vgId:%d, failed to auth since %s", vgId, tstrerror(code)); - return code; - } - - pVnode->accessState = pAuth->accessState; - dnodeReleaseVnode(pVnode); - return code; -} - -static int32_t vnodeProcessSyncVnodeReq(SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg); - - int32_t code = 0; - int32_t vgId = pAuth->vgId; - dDebug("vgId:%d, auth vnode req is received", vgId); - - SVnodeObj *pVnode = dnodeAcquireVnode(vgId); - if (pVnode == NULL) { - code = terrno; - dDebug("vgId:%d, failed to auth since %s", vgId, tstrerror(code)); - return code; - } - - code = vnodeSync(pVnode->pImpl); - if (code != 0) { - dError("vgId:%d, failed to auth vnode since %s", vgId, tstrerror(code)); - } - - dnodeReleaseVnode(pVnode); - return code; -} - -static int32_t vnodeProcessCompactVnodeReq(SRpcMsg *rpcMsg) { - SCompactVnodeMsg *pCompact = (SCompactVnodeMsg *)vnodeParseDropVnodeReq(rpcMsg); - - int32_t code = 0; - int32_t vgId = pCompact->vgId; - dDebug("vgId:%d, compact vnode req is received", vgId); - - SVnodeObj *pVnode = dnodeAcquireVnode(vgId); - if (pVnode == NULL) { - code = terrno; - dDebug("vgId:%d, failed to compact since %s", vgId, tstrerror(code)); - return code; - } - - code = vnodeCompact(pVnode->pImpl); - if (code != 0) { - dError("vgId:%d, failed to compact vnode since %s", vgId, tstrerror(code)); - } - - dnodeReleaseVnode(pVnode); - return code; -} - -static void dnodeProcessVnodeMgmtQueue(void *unused, SRpcMsg *pMsg) { - int32_t code = 0; - - switch (pMsg->msgType) { - case TSDB_MSG_TYPE_CREATE_VNODE_IN: - code = vnodeProcessCreateVnodeReq(pMsg); - break; - case TSDB_MSG_TYPE_ALTER_VNODE_IN: - code = vnodeProcessAlterVnodeReq(pMsg); - break; - case TSDB_MSG_TYPE_DROP_VNODE_IN: - code = vnodeProcessDropVnodeReq(pMsg); - break; - case TSDB_MSG_TYPE_AUTH_VNODE_IN: - code = vnodeProcessAuthVnodeReq(pMsg); - break; - case TSDB_MSG_TYPE_SYNC_VNODE_IN: - code = vnodeProcessSyncVnodeReq(pMsg); - break; - case TSDB_MSG_TYPE_COMPACT_VNODE_IN: - code = vnodeProcessCompactVnodeReq(pMsg); - break; - default: - code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - break; - } - - SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; - rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void dnodeProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY); -} - -static void dnodeProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH); -} - -static void dnodeProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs); - SRpcMsg *pRpcMsg = NULL; - - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - vnodeAppendMsg(pMsg, pRpcMsg); - taosFreeQitem(pRpcMsg); - } - - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE); -} - -static void dnodeProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = NULL; - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pMsg); - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY); - } -} - -static void dnodeProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = NULL; - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pMsg); - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC); - } -} - -static int32_t dnodeWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { - int32_t code = 0; - - if (pQueue == NULL) { - code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - } else { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); - if (pMsg == NULL) { - code = TSDB_CODE_DND_OUT_OF_MEMORY; - } else { - *pMsg = *pRpcMsg; - code = taosWriteQitem(pQueue, pMsg); - } - } - - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); - rpcFreeCont(pRpcMsg->pCont); - } -} - -static int32_t dnodeWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { - int32_t code = 0; - - if (pQueue == NULL) { - code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - } else { - SVnodeMsg *pMsg = vnodeInitMsg(1); - if (pMsg == NULL) { - code = TSDB_CODE_DND_OUT_OF_MEMORY; - } else { - vnodeAppendMsg(pMsg, pRpcMsg); - code = taosWriteQitem(pQueue, pMsg); - } - } - - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); - rpcFreeCont(pRpcMsg->pCont); - } -} - -static SVnodeObj *dnodeAcquireVnodeFromMsg(SRpcMsg *pMsg) { - SMsgHead *pHead = (SMsgHead *)pMsg->pCont; - pHead->vgId = htonl(pHead->vgId); - - SVnodeObj *pVnode = dnodeAcquireVnode(pHead->vgId); - if (pVnode == NULL) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); - } - - return pVnode; -} - -void dnodeProcessVnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { dnodeWriteRpcMsgToVnodeQueue(tsVnodes.pMgmtQ, pMsg); } - -void dnodeProcessVnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg); - if (pVnode != NULL) { - dnodeWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg); - dnodeReleaseVnode(pVnode); - } -} - -void dnodeProcessVnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg); - if (pVnode != NULL) { - dnodeWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg); - dnodeReleaseVnode(pVnode); - } -} - -void dnodeProcessVnodeQueryMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg); - if (pVnode != NULL) { - dnodeWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg); - dnodeReleaseVnode(pVnode); - } -} - -void dnodeProcessVnodeFetchMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { - SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg); - if (pVnode != NULL) { - dnodeWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg); - dnodeReleaseVnode(pVnode); - } -} - -static int32_t dnodePutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg) { - SVnodeObj *pVnode = dnodeAcquireVnode(vgId); - if (pVnode == NULL) { - return terrno; - } - - int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg); - dnodeReleaseVnode(pVnode); - return code; -} - -static int32_t dnodeInitVnodeMgmtWorker() { - SWorkerPool *pPool = &tsVnodes.mgmtPool; - pPool->name = "vnode-mgmt"; - pPool->min = 1; - pPool->max = 1; - if (tWorkerInit(pPool) != 0) { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - tsVnodes.pMgmtQ = tWorkerAllocQueue(pPool, NULL, (FProcessItem)dnodeProcessVnodeMgmtQueue); - if (tsVnodes.pMgmtQ == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - return 0; -} - -static void dnodeCleanupVnodeMgmtWorker() { - tWorkerFreeQueue(&tsVnodes.mgmtPool, tsVnodes.pMgmtQ); - tWorkerCleanup(&tsVnodes.mgmtPool); - tsVnodes.pMgmtQ = NULL; -} - -static int32_t dnodeAllocVnodeQueryQueue(SVnodeObj *pVnode) { - pVnode->pQueryQ = tWorkerAllocQueue(&tsVnodes.queryPool, pVnode, (FProcessItem)dnodeProcessVnodeQueryQueue); - if (pVnode->pQueryQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeVnodeQueryQueue(SVnodeObj *pVnode) { - tWorkerFreeQueue(&tsVnodes.queryPool, pVnode->pQueryQ); - pVnode->pQueryQ = NULL; -} - -static int32_t dnodeAllocVnodeFetchQueue(SVnodeObj *pVnode) { - pVnode->pFetchQ = tWorkerAllocQueue(&tsVnodes.fetchPool, pVnode, (FProcessItem)dnodeProcessVnodeFetchQueue); - if (pVnode->pFetchQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeVnodeFetchQueue(SVnodeObj *pVnode) { - tWorkerFreeQueue(&tsVnodes.fetchPool, pVnode->pFetchQ); - pVnode->pFetchQ = NULL; -} - -static int32_t dnodeInitVnodeReadWorker() { - int32_t maxFetchThreads = 4; - float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1); - - SWorkerPool *pPool = &tsVnodes.queryPool; - pPool->name = "vnode-query"; - pPool->min = (int32_t)threadsForQuery; - pPool->max = pPool->min; - if (tWorkerInit(pPool) != 0) { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - pPool = &tsVnodes.fetchPool; - pPool->name = "vnode-fetch"; - pPool->min = MIN(maxFetchThreads, tsNumOfCores); - pPool->max = pPool->min; - if (tWorkerInit(pPool) != 0) { - TSDB_CODE_VND_OUT_OF_MEMORY; - } - - return 0; -} - -static void dnodeCleanupVnodeReadWorker() { - tWorkerCleanup(&tsVnodes.fetchPool); - tWorkerCleanup(&tsVnodes.queryPool); -} - -static int32_t dnodeAllocVnodeWriteQueue(SVnodeObj *pVnode) { - pVnode->pWriteQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeWriteQueue); - if (pVnode->pWriteQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeVnodeWriteQueue(SVnodeObj *pVnode) { - tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pWriteQ); - pVnode->pWriteQ = NULL; -} - -static int32_t dnodeAllocVnodeApplyQueue(SVnodeObj *pVnode) { - pVnode->pApplyQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeApplyQueue); - if (pVnode->pApplyQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeVnodeApplyQueue(SVnodeObj *pVnode) { - tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pApplyQ); - pVnode->pApplyQ = NULL; -} - -static int32_t dnodeInitVnodeWriteWorker() { - SMWorkerPool *pPool = &tsVnodes.writePool; - pPool->name = "vnode-write"; - pPool->max = tsNumOfCores; - if (tMWorkerInit(pPool) != 0) { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - return 0; -} - -static void dnodeCleanupVnodeWriteWorker() { tMWorkerCleanup(&tsVnodes.writePool); } - -static int32_t dnodeAllocVnodeSyncQueue(SVnodeObj *pVnode) { - pVnode->pSyncQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeSyncQueue); - if (pVnode->pSyncQ == NULL) { - return TSDB_CODE_DND_OUT_OF_MEMORY; - } - return 0; -} - -static void dnodeFreeVnodeSyncQueue(SVnodeObj *pVnode) { - tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pSyncQ); - pVnode->pSyncQ = NULL; -} - -static int32_t dnodeInitVnodeSyncWorker() { - int32_t maxThreads = tsNumOfCores / 2; - if (maxThreads < 1) maxThreads = 1; - - SMWorkerPool *pPool = &tsVnodes.writePool; - pPool->name = "vnode-sync"; - pPool->max = maxThreads; - if (tMWorkerInit(pPool) != 0) { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - return 0; -} - -static void dnodeCleanupVnodeSyncWorker() { tMWorkerCleanup(&tsVnodes.syncPool); } - -static int32_t dnodeInitVnodeModule() { - SVnodePara para; - para.sendMsgToDnodeFp = dnodeSendMsgToDnode; - para.sendMsgToMnodeFp = dnodeSendMsgToMnode; - para.putMsgToApplyQueueFp = dnodePutMsgIntoVnodeApplyQueue; - - return vnodeInit(para); -} - -int32_t dnodeInitVnodes() { - dInfo("dnode-vnodes start to init"); - - SSteps *pSteps = taosStepInit(6, dnodeReportStartup); - taosStepAdd(pSteps, "dnode-vnode-env", dnodeInitVnodeModule, vnodeCleanup); - taosStepAdd(pSteps, "dnode-vnode-mgmt", dnodeInitVnodeMgmtWorker, dnodeCleanupVnodeMgmtWorker); - taosStepAdd(pSteps, "dnode-vnode-read", dnodeInitVnodeReadWorker, dnodeCleanupVnodeReadWorker); - taosStepAdd(pSteps, "dnode-vnode-write", dnodeInitVnodeWriteWorker, dnodeCleanupVnodeWriteWorker); - taosStepAdd(pSteps, "dnode-vnode-sync", dnodeInitVnodeSyncWorker, dnodeCleanupVnodeSyncWorker); - taosStepAdd(pSteps, "dnode-vnodes", dnodeOpenVnodes, dnodeCleanupVnodes); - - tsVnodes.pSteps = pSteps; - return taosStepExec(pSteps); -} - -void dnodeCleanupVnodes() { - if (tsVnodes.pSteps != NULL) { - dInfo("dnode-vnodes start to clean up"); - taosStepCleanup(tsVnodes.pSteps); - tsVnodes.pSteps = NULL; - dInfo("dnode-vnodes is cleaned up"); - } -} - -void dnodeGetVnodeLoads(SVnodeLoads *pLoads) { - pLoads->num = taosHashGetSize(tsVnodes.hash); - - int32_t v = 0; - void *pIter = taosHashIterate(tsVnodes.hash, NULL); - while (pIter) { - SVnodeObj **ppVnode = pIter; - if (ppVnode == NULL) continue; - - SVnodeObj *pVnode = *ppVnode; - if (pVnode == NULL) continue; - - SVnodeLoad *pLoad = &pLoads->data[v++]; - vnodeGetLoad(pVnode->pImpl, pLoad); - pLoad->vgId = htonl(pLoad->vgId); - pLoad->totalStorage = htobe64(pLoad->totalStorage); - pLoad->compStorage = htobe64(pLoad->compStorage); - pLoad->pointsWritten = htobe64(pLoad->pointsWritten); - pLoad->tablesNum = htobe64(pLoad->tablesNum); - - pIter = taosHashIterate(tsVnodes.hash, pIter); - } -} diff --git a/source/dnode/mnode/impl/inc/mnodeDef.h b/source/dnode/mnode/impl/inc/mnodeDef.h index ccdba13006..4b4c4abdb3 100644 --- a/source/dnode/mnode/impl/inc/mnodeDef.h +++ b/source/dnode/mnode/impl/inc/mnodeDef.h @@ -131,7 +131,7 @@ typedef struct SMnodeObj { int64_t roleTime; int64_t createdTime; int64_t updateTime; - SDnodeObj *pDnode; + SDnodeObj *pDnd; } SMnodeObj; typedef struct { @@ -215,7 +215,7 @@ typedef struct SDbObj { typedef struct { int32_t dnodeId; int8_t role; - SDnodeObj *pDnode; + SDnodeObj *pDnd; } SVnodeGid; typedef struct SVgObj { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index bc4718ee5b..877ec1431c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -83,7 +83,7 @@ static int32_t mnodeSetOptions(SMnode *pMnode, const SMnodeOptions *pOptions) { pMnode->replica = pOptions->replica; pMnode->selfIndex = pOptions->selfIndex; memcpy(&pMnode->replicas, pOptions->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); - pMnode->pServer = pOptions->pServer; + pMnode->pServer = pOptions->pDnode; pMnode->putMsgToApplyMsgFp = pOptions->putMsgToApplyMsgFp; pMnode->sendMsgToDnodeFp = pOptions->sendMsgToDnodeFp; pMnode->sendMsgToMnodeFp = pOptions->sendMsgToMnodeFp; @@ -187,7 +187,7 @@ SMnodeMsg *mnodeInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { } if (rpcGetConnInfo(pRpcMsg->handle, &pMsg->conn) != 0) { - mnodeCleanupMsg(pMnode, pMsg); + mnodeCleanupMsg(pMsg); mError("can not get user from conn:%p", pMsg->rpcMsg.handle); terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; return NULL; @@ -199,7 +199,7 @@ SMnodeMsg *mnodeInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return pMsg; } -void mnodeCleanupMsg(SMnode *pMnode, SMnodeMsg *pMsg) { +void mnodeCleanupMsg(SMnodeMsg *pMsg) { if (pMsg->pUser != NULL) { sdbRelease(pMsg->pUser); } @@ -210,7 +210,7 @@ void mnodeCleanupMsg(SMnode *pMnode, SMnodeMsg *pMsg) { static void mnodeProcessRpcMsg(SMnodeMsg *pMsg) { if (!mnodeIsMaster()) { mnodeSendRedirectMsg(NULL, &pMsg->rpcMsg, true); - mnodeCleanupMsg(NULL, pMsg); + mnodeCleanupMsg(pMsg); return; } diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 934a8dd6ab..a2041c76fc 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -54,10 +54,11 @@ typedef struct { char secret[TSDB_KEY_LEN]; // secret for the link char ckey[TSDB_KEY_LEN]; // ciphering key - void (*cfp)(SRpcMsg *, SEpSet *); - int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); + void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t refCount; + void *parent; void *idPool; // handle to ID pool void *tmrCtrl; // handle to timer SHashObj *hash; // handle returned by hash utility @@ -260,6 +261,7 @@ void *rpcOpen(const SRpcInit *pInit) { pRpc->spi = pInit->spi; pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; + pRpc->parent = pInit->parent; pRpc->refCount = 1; atomic_add_fetch_32(&tsRpcNum, 1); @@ -744,7 +746,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { if (pConn->user[0] == 0) { terrno = TSDB_CODE_RPC_AUTH_REQUIRED; } else { - terrno = (*pRpc->afp)(pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); + terrno = (*pRpc->afp)(pRpc->parent, pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); } if (terrno != 0) { @@ -1024,8 +1026,8 @@ static void doRpcReportBrokenLinkToServer(void *param, void *id) { SRpcMsg *pRpcMsg = (SRpcMsg *)(param); SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); SRpcInfo *pRpc = pConn->pRpc; - (*(pRpc->cfp))(pRpcMsg, NULL); - free(pRpcMsg); + (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL); + free(pRpcMsg); } static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; @@ -1137,9 +1139,9 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { // for asynchronous API SEpSet *pEpSet = NULL; if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) - pEpSet = &pContext->epSet; + pEpSet = &pContext->epSet; - (*pRpc->cfp)(pMsg, pEpSet); + (*pRpc->cfp)(pRpc->parent, pMsg, pEpSet); } // free the request message @@ -1155,15 +1157,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; rpcMsg.msgType = pHead->msgType; - rpcMsg.code = pHead->code; - - if ( rpcIsReq(pHead->msgType) ) { + rpcMsg.code = pHead->code; + + if (rpcIsReq(pHead->msgType)) { rpcMsg.ahandle = pConn->ahandle; rpcMsg.handle = pConn; rpcAddRef(pRpc); // add the refCount for requests // notify the server app - (*(pRpc->cfp))(&rpcMsg, NULL); + (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); } else { // it's a response rpcMsg.handle = pContext; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0b10274c00..6d4f4eb6a7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -78,6 +78,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message") +TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") @@ -235,20 +236,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_PARTITONS, "Invalid topic partito TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") // dnode -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, "Message not processed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, "Dnode out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE, "Mnode Id not match Dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED, "Mnode already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_NOT_DEPLOYED, "Mnode not deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_READ_MNODE_FILE_ERROR, "Read mnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR, "Write mnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, "No permission for disk files in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES, "Too many vnode directories") TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR, "Parse vnodes.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_PARSE_DNODE_FILE_ERROR, "Parse dnodes.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_READ_FILE_ERROR, "Read dnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR, "Write dnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED, "Mnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_NOT_DEPLOYED, "Mnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_INVALID, "Mnode Id invalid") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_NOT_FOUND, "Mnode Id not found") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_READ_FILE_ERROR, "Read mnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR, "Write mnode.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnode directories") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR, "Read vnodes.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR, "Write vnodes.json error") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, "Action in progress") -- GitLab