From 053280da8a1643aed656f803e715e4645f789052 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 18 Mar 2020 17:31:52 +0800 Subject: [PATCH] [TD-10] add dnode status message --- src/dnode/inc/dnodeMClient.h | 6 +- src/dnode/inc/dnodeMgmt.h | 1 + src/dnode/inc/dnodeModule.h | 1 + src/dnode/src/dnodeMClient.c | 131 ++++++++++++++++++-- src/dnode/src/dnodeMgmt.c | 234 +++++++++++++++++++++++------------ src/dnode/src/dnodeMnode.c | 7 +- src/dnode/src/dnodeModule.c | 35 +++++- src/dnode/src/dnodeWrite.c | 2 +- src/inc/mnode.h | 1 + src/inc/taosdef.h | 1 + src/inc/taosmsg.h | 27 ++-- src/mnode/inc/mgmtVgroup.h | 6 +- src/mnode/src/mgmtDServer.c | 60 +-------- src/mnode/src/mgmtDnode.c | 162 +++++++++++++++++++++--- src/mnode/src/mgmtMnode.c | 3 +- src/mnode/src/mgmtShell.c | 2 +- src/mnode/src/mgmtVgroup.c | 55 +++++--- src/util/inc/ihash.h | 2 +- src/util/src/ihash.c | 15 +-- src/util/src/tstatus.c | 14 +-- src/vnode/tsdb/inc/tsdb.h | 1 - 21 files changed, 534 insertions(+), 232 deletions(-) diff --git a/src/dnode/inc/dnodeMClient.h b/src/dnode/inc/dnodeMClient.h index 391e8da2c1..cab9ea9be4 100644 --- a/src/dnode/inc/dnodeMClient.h +++ b/src/dnode/inc/dnodeMClient.h @@ -20,8 +20,10 @@ extern "C" { #endif -int32_t dnodeInitMClient(); -void dnodeCleanupMClient(); +int32_t dnodeInitMClient(); +void dnodeCleanupMClient(); +void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); +uint32_t dnodeGetMnodeMasteIp(); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h index b8d01916fe..f944bd5add 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/dnode/inc/dnodeMgmt.h @@ -23,6 +23,7 @@ extern "C" { int32_t dnodeInitMgmt(); void dnodeCleanupMgmt(); void dnodeMgmt(SRpcMsg *rpcMsg); +void dnodeUpdateDnodeId(int32_t dnodeId); void* dnodeGetVnode(int32_t vgId); int32_t dnodeGetVnodeStatus(void *pVnode); diff --git a/src/dnode/inc/dnodeModule.h b/src/dnode/inc/dnodeModule.h index 3e8e7d28f5..728630748f 100644 --- a/src/dnode/inc/dnodeModule.h +++ b/src/dnode/inc/dnodeModule.h @@ -23,6 +23,7 @@ extern "C" { int32_t dnodeInitModules(); void dnodeCleanUpModules(); void dnodeStartModules(); +void dnodeProcessModuleStatus(uint32_t moduleStatus); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index 226a47f975..e3a7f99a5f 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -18,16 +18,22 @@ #include "taosmsg.h" #include "tlog.h" #include "trpc.h" +#include "tutil.h" #include "dnode.h" +#include "dnodeMClient.h" -static void (*dnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); +static bool dnodeReadMnodeIpList(); +static void dnodeSaveMnodeIpList(); static void dnodeProcessRspFromMnode(SRpcMsg *pMsg); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); -static void *tsDnodeMClientRpc; +static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); +static void *tsDnodeMClientRpc = NULL; +static SRpcIpSet tsDnodeMnodeIpList = {0}; int32_t dnodeInitMClient() { - dnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp; - + dnodeReadMnodeIpList(); + tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp; + SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; @@ -35,9 +41,12 @@ int32_t dnodeInitMClient() { rpcInit.label = "DND-MC"; rpcInit.numOfThreads = 1; rpcInit.cfp = dnodeProcessRspFromMnode; - rpcInit.sessions = TSDB_SESSIONS_PER_DNODE; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 2000; + rpcInit.user = "t"; + rpcInit.ckey = "key"; + rpcInit.secret = "secret"; tsDnodeMClientRpc = rpcOpen(&rpcInit); if (tsDnodeMClientRpc == NULL) { @@ -53,18 +62,122 @@ void dnodeCleanupMClient() { if (tsDnodeMClientRpc) { rpcClose(tsDnodeMClientRpc); tsDnodeMClientRpc = NULL; + dPrint("mnode rpc client is closed"); } } static void dnodeProcessRspFromMnode(SRpcMsg *pMsg) { - if (dnodeProcessMgmtRspFp[pMsg->msgType]) { - (*dnodeProcessMgmtRspFp[pMsg->msgType])(pMsg); + if (tsDnodeProcessMgmtRspFp[pMsg->msgType]) { + (*tsDnodeProcessMgmtRspFp[pMsg->msgType])(pMsg); } else { - dError("%s is not processed in mclient", taosMsg[pMsg->msgType]); + dError("%s is not processed in mnode rpc client", taosMsg[pMsg->msgType]); } rpcFreeCont(pMsg->pCont); } static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { + if (pMsg->code != TSDB_CODE_SUCCESS) { + dError("status rsp is received, reason:%s", tstrerror(pMsg->code)); + return; + } + + SDMStatusRsp *pStatusRsp = pMsg->pCont; + if (pStatusRsp->ipList.numOfIps <= 0) { + dError("status msg is invalid, num of ips is %d", pStatusRsp->ipList.numOfIps); + return; + } + + pStatusRsp->ipList.port = htons(pStatusRsp->ipList.port); + for (int32_t i = 0; i < pStatusRsp->ipList.numOfIps; ++i) { + pStatusRsp->ipList.ip[i] = htonl(pStatusRsp->ipList.ip[i]); + } + + dTrace("status msg is received, result:%d", tstrerror(pMsg->code)); + + if (memcmp(&(pStatusRsp->ipList), &tsDnodeMnodeIpList, sizeof(SRpcIpSet)) != 0) { + dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", pStatusRsp->ipList.numOfIps, pStatusRsp->ipList.inUse); + memcpy(&tsDnodeMnodeIpList, &pStatusRsp->ipList, sizeof(SRpcIpSet)); + for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; ++i) { + dPrint("mnode IP index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i])); + } + dnodeSaveMnodeIpList(); + } + + SDnodeState *pState = &pStatusRsp->dnodeState; + pState->numOfVnodes = htonl(pState->numOfVnodes); + pState->moduleStatus = htonl(pState->moduleStatus); + pState->createdTime = htonl(pState->createdTime); + pState->dnodeId = htonl(pState->dnodeId); + + dnodeProcessModuleStatus(pState->moduleStatus); + dnodeUpdateDnodeId(pState->dnodeId); } + +void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { + rpcSendRequest(tsDnodeMClientRpc, &tsDnodeMnodeIpList, rpcMsg); +} + +static bool dnodeReadMnodeIpList() { + char ipFile[TSDB_FILENAME_LEN] = {0}; + sprintf(ipFile, "%s/iplist", tsDnodeDir); + + FILE *fp = fopen(ipFile, "r"); + if (!fp) return false; + + char option[32] = {0}; + int32_t value = 0; + int32_t num = 0; + + fscanf(fp, "%s %d", option, &value); + if (num != 2) return false; + if (strcmp(option, "inUse") != 0) return false; + tsDnodeMnodeIpList.inUse = (int8_t)value;; + + num = fscanf(fp, "%s %d", option, &value); + if (num != 2) return false; + if (strcmp(option, "numOfIps") != 0) return false; + tsDnodeMnodeIpList.numOfIps = (int8_t)value; + + num = fscanf(fp, "%s %d", option, &value); + if (num != 2) return false; + if (strcmp(option, "port") != 0) return false; + tsDnodeMnodeIpList.port = (uint16_t)value; + + for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { + num = fscanf(fp, "%s %d", option, &value); + if (num != 2) return false; + if (strncmp(option, "ip", 2) != 0) return false; + tsDnodeMnodeIpList.ip[i] = (uint32_t)value; + } + + fclose(fp); + dPrint("read mnode iplist successed"); + for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { + dPrint("mnode IP index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i])); + } + + return true; +} + +static void dnodeSaveMnodeIpList() { + char ipFile[TSDB_FILENAME_LEN] = {0}; + sprintf(ipFile, "%s/iplist", tsDnodeDir); + + FILE *fp = fopen(ipFile, "w"); + if (!fp) return; + + fprintf(fp, "inUse %d\n", tsDnodeMnodeIpList.inUse); + fprintf(fp, "numOfIps %d\n", tsDnodeMnodeIpList.numOfIps); + fprintf(fp, "port %u\n", tsDnodeMnodeIpList.port); + for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { + fprintf(fp, "ip%d %u\n", i, tsDnodeMnodeIpList.ip[i]); + } + + fclose(fp); + dPrint("save mnode iplist successed"); +} + +uint32_t dnodeGetMnodeMasteIp() { + return tsDnodeMnodeIpList.ip[0]; +} \ No newline at end of file diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index e547faa7a0..caff618048 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -22,16 +22,18 @@ #include "trpc.h" #include "tstatus.h" #include "tsdb.h" +#include "ttime.h" #include "ttimer.h" +#include "dnodeMClient.h" #include "dnodeMgmt.h" #include "dnodeRead.h" #include "dnodeWrite.h" typedef struct { int32_t vgId; // global vnode group ID - int32_t vnode; - int32_t status; // status: master, slave, notready, deleting int32_t refCount; // reference count + int8_t dirty; + int8_t status; // status: master, slave, notready, deleting int64_t version; void *wworker; void *rworker; @@ -48,6 +50,7 @@ static int32_t dnodeOpenVnode(int32_t vnode, char *rootDir); static void dnodeCleanupVnode(SVnodeObj *pVnode); static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *cfg); static void dnodeDropVnode(SVnodeObj *pVnode); +static void dnodeDoDropVnode(SVnodeObj *pVnode); static void dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg); static void dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); static void dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); @@ -55,17 +58,23 @@ static void dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); static void dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); static void (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); static void dnodeSendStatusMsg(void *handle, void *tmrId); +static void dnodeReadDnodeId(); -static void * tsDnodeVnodesHash = NULL; -static void *tsDnodeTmr = NULL; -static void *tsStatusTimer = NULL; +static void *tsDnodeVnodesHash = NULL; +static void *tsDnodeTmr = NULL; +static void *tsStatusTimer = NULL; +static uint32_t tsRebootTime; +static int32_t tsDnodeId = 0; +static char tsDnodeName[TSDB_DNODE_NAME_LEN]; int32_t dnodeInitMgmt() { + dnodeReadDnodeId(); + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; tsDnodeVnodesHash = taosInitIntHash(TSDB_MAX_VNODES, sizeof(SVnodeObj), taosHashInt); if (tsDnodeVnodesHash == NULL) { @@ -73,6 +82,8 @@ int32_t dnodeInitMgmt() { return -1; } + tsRebootTime = taosGetTimestampSec(); + tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); if (tsDnodeTmr == NULL) { dError("failed to init dnode timer"); @@ -89,8 +100,16 @@ void dnodeCleanupMgmt() { tsStatusTimer = NULL; } + if (tsDnodeTmr != NULL) { + taosTmrCleanUp(tsDnodeTmr); + tsDnodeTmr = NULL; + } + dnodeCleanupVnodes(); - taosCleanUpIntHash(tsDnodeVnodesHash); + if (tsDnodeVnodesHash == NULL) { + taosCleanUpIntHash(tsDnodeVnodesHash); + tsDnodeVnodesHash = NULL; + } } void dnodeMgmt(SRpcMsg *pMsg) { @@ -106,7 +125,7 @@ void dnodeMgmt(SRpcMsg *pMsg) { rpcSendResponse(&rsp); } - rpcFreeCont(pMsg->pCont); // free the received message + rpcFreeCont(pMsg->pCont); } void *dnodeGetVnode(int32_t vgId) { @@ -145,8 +164,13 @@ void *dnodeGetVnodeTsdb(void *pVnode) { return ((SVnodeObj *)pVnode)->tsdb; } -void dnodeReleaseVnode(void *pVnode) { - atomic_sub_fetch_32(&((SVnodeObj *) pVnode)->refCount, 1); +void dnodeReleaseVnode(void *pVnodeRaw) { + SVnodeObj *pVnode = pVnodeRaw; + int32_t count = atomic_sub_fetch_32(&pVnode->refCount, 1); + + if (count == 0 && pVnode->dirty) { + dnodeDoDropVnode(pVnode); + } } static int32_t dnodeOpenVnodes() { @@ -194,11 +218,10 @@ static int32_t dnodeOpenVnode(int32_t vnode, char *rootDir) { return terrno; } - //STsdbRepoInfo *tsdbInfo = tsdbGetStatus(pTsdb); + STsdbRepoInfo *tsdbInfo = tsdbGetStatus(pTsdb); - SVnodeObj vnodeObj; - vnodeObj.vgId = vnode; //tsdbInfo->tsdbCfg.vgId; - vnodeObj.vnode = vnode; //tsdbInfo->tsdbCfg.tsdbId; + SVnodeObj vnodeObj = {0}; + vnodeObj.vgId = tsdbInfo->tsdbCfg.tsdbId; vnodeObj.status = TSDB_VN_STATUS_NOT_READY; vnodeObj.refCount = 1; vnodeObj.version = 0; @@ -212,7 +235,7 @@ static int32_t dnodeOpenVnode(int32_t vnode, char *rootDir) { taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj)); - dTrace("open vnode:%d in %s", vnodeObj.vnode, rootDir); + dTrace("open vnode:%d in %s", vnodeObj.vgId, rootDir); return TSDB_CODE_SUCCESS; } @@ -241,14 +264,13 @@ static void dnodeCleanupVnode(SVnodeObj *pVnode) { pVnode->tsdb = NULL; } - dTrace("cleanup vnode:%d", pVnode->vnode); + dTrace("cleanup vnode:%d", pVnode->vgId); } static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) { STsdbCfg tsdbCfg = {0}; - tsdbCfg.vgId = pVnodeCfg->cfg.vgId; tsdbCfg.precision = pVnodeCfg->cfg.precision; - tsdbCfg.tsdbId = pVnodeCfg->vnode; + tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; tsdbCfg.maxTables = pVnodeCfg->cfg.maxSessions; tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; tsdbCfg.minRowsPerFileBlock = -1; @@ -283,13 +305,12 @@ static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) { void *pTsdb = tsdbCreateRepo(rootDir, &tsdbCfg, NULL); if (pTsdb == NULL) { - dError("vgroup:%d, failed to create tsdb in vnode:%d, reason:%s", pVnodeCfg->cfg.vgId, pVnodeCfg->vnode, tstrerror(terrno)); + dError("vgroup:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); return terrno; } - SVnodeObj vnodeObj; + SVnodeObj vnodeObj = {0}; vnodeObj.vgId = pVnodeCfg->cfg.vgId; - vnodeObj.vnode = pVnodeCfg->vnode; vnodeObj.status = TSDB_VN_STATUS_NOT_READY; vnodeObj.refCount = 1; vnodeObj.version = 0; @@ -303,18 +324,11 @@ static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) { taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj)); - dPrint("vgroup:%d, vnode:%d is created", pVnodeCfg->cfg.vgId, pVnodeCfg->vnode); + dPrint("vgroup:%d, vnode is created", vnodeObj.vgId); return TSDB_CODE_SUCCESS; } -static void dnodeDropVnode(SVnodeObj *pVnode) { - pVnode->status = TSDB_VN_STATUS_NOT_READY; - - int32_t count = atomic_sub_fetch_32(&pVnode->refCount, 1); - if (count > 0) { - // wait refcount - } - +static void dnodeDoDropVnode(SVnodeObj *pVnode) { if (pVnode->tsdb) { tsdbDropRepo(pVnode->tsdb); pVnode->tsdb = NULL; @@ -324,21 +338,33 @@ static void dnodeDropVnode(SVnodeObj *pVnode) { taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId); } +static void dnodeDropVnode(SVnodeObj *pVnode) { + pVnode->status = TSDB_VN_STATUS_NOT_READY; + pVnode->dirty = true; + + int32_t count = atomic_sub_fetch_32(&pVnode->refCount, 1); + if (count > 0) { + dTrace("vgroup:%d, vnode will be dropped until refcount:%d is 0", pVnode->vgId, count); + return; + } + + dnodeDoDropVnode(pVnode); +} + static void dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; - pCreate->vnode = htonl(pCreate->vnode); pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); pCreate->cfg.maxSessions = htonl(pCreate->cfg.maxSessions); pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); - dTrace("vgroup:%d, start to create vnode:%d", pCreate->cfg.vgId, pCreate->vnode); + dTrace("vgroup:%d, start to create vnode in dnode", pCreate->cfg.vgId); SVnodeObj *pVnodeObj = (SVnodeObj *) taosGetIntHashData(tsDnodeVnodesHash, pCreate->cfg.vgId); if (pVnodeObj != NULL) { rpcRsp.code = TSDB_CODE_SUCCESS; - dPrint("vgroup:%d, vnode:%d is already exist", pCreate->cfg.vgId, pCreate->vnode); + dPrint("vgroup:%d, vnode is already exist", pCreate->cfg.vgId); } else { rpcRsp.code = dnodeCreateVnode(pCreate); } @@ -352,6 +378,8 @@ static void dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { SMDDropVnodeMsg *pDrop = rpcMsg->pCont; pDrop->vgId = htonl(pDrop->vgId); + dTrace("vgroup:%d, start to drop vnode in dnode", pDrop->vgId); + SVnodeObj *pVnodeObj = (SVnodeObj *) taosGetIntHashData(tsDnodeVnodesHash, pDrop->vgId); if (pVnodeObj != NULL) { dnodeDropVnode(pVnodeObj); @@ -367,11 +395,12 @@ static void dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; - pCreate->vnode = htonl(pCreate->vnode); pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); pCreate->cfg.maxSessions = htonl(pCreate->cfg.maxSessions); pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); + dTrace("vgroup:%d, start to alter vnode in dnode", pCreate->cfg.vgId); + SVnodeObj *pVnodeObj = (SVnodeObj *) taosGetIntHashData(tsDnodeVnodesHash, pCreate->cfg.vgId); if (pVnodeObj != NULL) { rpcRsp.code = TSDB_CODE_SUCCESS; @@ -401,60 +430,103 @@ static void dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { rpcSendResponse(&rpcRsp); } +static void dnodeBuildVloadMsg(char *pNode, void * param) { + SVnodeObj *pVnode = (SVnodeObj *) pNode; + if (pVnode->dirty) return; + + SDMStatusMsg *pStatus = param; + if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; + + SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; + pLoad->vgId = htonl(pVnode->vgId); + pLoad->status = pVnode->status; +} + static void dnodeSendStatusMsg(void *handle, void *tmrId) { + if (tsDnodeTmr == NULL) { + dError("dnode timer is already released"); + return; + } + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); if (tsStatusTimer == NULL) { dError("failed to start status timer"); return; } -// int32_t contLen = sizeof(SDMStatusMsg) + dnodeGetVnodesNum() * sizeof(SVnodeLoad); -// SDMStatusMsg *pStatus = rpcMallocCont(contLen); -// if (pStatus == NULL) { -// dError("Failed to malloc status message"); -// return; -// } -// -// int32_t totalVnodes = dnodeGetVnodesNum(); -// -// pStatus->version = htonl(tsVersion); -// pStatus->privateIp = htonl(inet_addr(tsPrivateIp)); -// pStatus->publicIp = htonl(inet_addr(tsPublicIp)); -// pStatus->lastReboot = htonl(tsRebootTime); -// pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes); -// pStatus->openVnodes = htons((uint16_t) totalVnodes); -// pStatus->numOfCores = htons((uint16_t) tsNumOfCores); -// pStatus->diskAvailable = tsAvailDataDirGB; -// pStatus->alternativeRole = (uint8_t) tsAlternativeRole; -// -// SVnodeLoad *pLoad = (SVnodeLoad *)pStatus->load; - - //TODO loop all vnodes - // for (int32_t vnode = 0, count = 0; vnode <= totalVnodes; ++vnode) { - // if (vnodeList[vnode].cfg.maxSessions <= 0) continue; - // - // SVnodeObj *pVnode = vnodeList + vnode; - // pLoad->vnode = htonl(vnode); - // pLoad->vgId = htonl(pVnode->cfg.vgId); - // pLoad->status = (uint8_t)vnodeList[vnode].vnodeStatus; - // pLoad->syncStatus =(uint8_t)vnodeList[vnode].syncStatus; - // pLoad->accessState = (uint8_t)(pVnode->accessState); - // pLoad->totalStorage = htobe64(pVnode->vnodeStatistic.totalStorage); - // pLoad->compStorage = htobe64(pVnode->vnodeStatistic.compStorage); - // if (pVnode->vnodeStatus == TSDB_VN_STATUS_MASTER) { - // pLoad->pointsWritten = htobe64(pVnode->vnodeStatistic.pointsWritten); - // } else { - // pLoad->pointsWritten = htobe64(0); - // } - // pLoad++; - // - // if (++count >= tsOpenVnodes) { - // break; - // } - // } - -// dnodeSendMsgToMnode(TSDB_MSG_TYPE_STATUS, pStatus, contLen); + int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); + SDMStatusMsg *pStatus = rpcMallocCont(contLen); + if (pStatus == NULL) { + dError("failed to malloc status message"); + return; + } + + strcpy(pStatus->dnodeName, tsDnodeName); + pStatus->version = htonl(tsVersion); + pStatus->dnodeId = htonl(tsDnodeId); + pStatus->privateIp = htonl(inet_addr(tsPrivateIp)); + pStatus->publicIp = htonl(inet_addr(tsPublicIp)); + pStatus->lastReboot = htonl(tsRebootTime); + pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes); + pStatus->numOfCores = htons((uint16_t) tsNumOfCores); + pStatus->diskAvailable = tsAvailDataDirGB; + pStatus->alternativeRole = (uint8_t) tsAlternativeRole; + + taosVisitIntHashWithFp(tsDnodeVnodesHash, dnodeBuildVloadMsg, pStatus); + contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); + pStatus->openVnodes = htons(pStatus->openVnodes); + + SRpcMsg rpcMsg = { + .pCont = pStatus, + .contLen = contLen, + .msgType = TSDB_MSG_TYPE_DM_STATUS + }; + + dnodeSendMsgToMnode(&rpcMsg); +} + +static void dnodeReadDnodeId() { + char dnodeIdFile[TSDB_FILENAME_LEN] = {0}; + sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir); + + FILE *fp = fopen(dnodeIdFile, "r"); + if (!fp) return; + + char option[32] = {0}; + int32_t value = 0; + int32_t num = 0; + + fscanf(fp, "%s %d", option, &value); + if (num != 2) return false; + if (strcmp(option, "dnodeId") != 0) return false; + tsDnodeId = value;; + + fclose(fp); + dPrint("read dnodeId:%d successed", tsDnodeId); } +static void dnodeSaveDnodeId() { + char dnodeIdFile[TSDB_FILENAME_LEN] = {0}; + sprintf(dnodeIdFile, "%s/dnodeId", tsDnodeDir); + + FILE *fp = fopen(dnodeIdFile, "w"); + if (!fp) { + return false; + } + + fprintf(fp, "dnodeId %d\n", tsDnodeId); + + fclose(fp); + dPrint("save dnodeId successed"); + + return true; +} +void dnodeUpdateDnodeId(int32_t dnodeId) { + if (tsDnodeId == 0) { + dPrint("dnodeId is set to %d", dnodeId); + tsDnodeId = dnodeId; + dnodeSaveDnodeId(); + } +} diff --git a/src/dnode/src/dnodeMnode.c b/src/dnode/src/dnodeMnode.c index da47e4e49c..9d1be0148e 100644 --- a/src/dnode/src/dnodeMnode.c +++ b/src/dnode/src/dnodeMnode.c @@ -40,12 +40,12 @@ int32_t dnodeInitMnode() { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; rpcInit.localPort = tsDnodeMnodePort; - rpcInit.label = "DND-mnode"; + rpcInit.label = "DND-MS"; rpcInit.numOfThreads = 1; rpcInit.cfp = dnodeProcessMsgFromMnode; - rpcInit.sessions = TSDB_SESSIONS_PER_DNODE; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 2000; tsDnodeMnodeRpc = rpcOpen(&rpcInit); if (tsDnodeMnodeRpc == NULL) { @@ -61,6 +61,7 @@ void dnodeCleanupMnode() { if (tsDnodeMnodeRpc) { rpcClose(tsDnodeMnodeRpc); tsDnodeMnodeRpc = NULL; + dPrint("mnode rpc server is closed"); } } diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 9f5164dd5b..6835f63793 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -74,7 +74,7 @@ int32_t dnodeInitModules() { for (int mod = 0; mod < TSDB_MOD_MAX; ++mod) { if (tsModule[mod].num != 0 && tsModule[mod].initFp) { if ((*tsModule[mod].initFp)() != 0) { - dError("TDengine initialization failed"); + dError("failed to init modules"); return -1; } } @@ -92,3 +92,36 @@ void dnodeStartModules() { } } } + +void dnodeProcessModuleStatus(uint32_t moduleStatus) { + if (moduleStatus == tsModuleStatus) return; + + dPrint("module status is received, old:%d, new:%d", tsModuleStatus, moduleStatus); + + int news = moduleStatus; + int olds = tsModuleStatus; + + for (int moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { + int newStatus = news & (1 << moduleType); + int oldStatus = olds & (1 << moduleType); + + if (oldStatus > 0) { + if (newStatus == 0) { + if (tsModule[moduleType].stopFp) { + dPrint("module:%s is stopped on this node", tsModule[moduleType].name); + (*tsModule[moduleType].stopFp)(); + } + } + } else if (oldStatus == 0) { + if (newStatus > 0) { + if (tsModule[moduleType].startFp) { + dPrint("module:%s is started on this node", tsModule[moduleType].name); + (*tsModule[moduleType].startFp)(); + } + } + } else { + } + } + + tsModuleStatus = moduleStatus; +} diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index b10ca16467..1deefd4f53 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -94,7 +94,7 @@ void dnodeWrite(SRpcMsg *pMsg) { SRpcContext *pRpcContext = NULL; if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT || pMsg->msgType == TSDB_MSG_TYPE_MD_DROP_STABLE) { - SMsgDesc *pDesc = pCont; + SMsgDesc *pDesc = (SMsgDesc *)pCont; pDesc->numOfVnodes = htonl(pDesc->numOfVnodes); pCont += sizeof(SMsgDesc); if (pDesc->numOfVnodes > 1) { diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 48aeb2dfe6..d42d2f45f0 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -40,6 +40,7 @@ extern "C" { #include "tutil.h" typedef struct { + int32_t dnodeId; uint32_t privateIp; int32_t sid; uint32_t moduleStatus; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 712c99adc2..7aac36786a 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -165,6 +165,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_COLUMNS 256 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns +#define TSDB_DNODE_NAME_LEN 63 #define TSDB_TABLE_NAME_LEN 64 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index e4b083bb9d..38fcb3fd42 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "taoserror.h" -#include "taosdef.h" #include "trpc.h" // message type @@ -520,16 +519,14 @@ typedef struct { } SRetrieveTableRsp; typedef struct { - uint32_t vnode; - uint32_t vgId; - uint8_t status; - uint8_t dropStatus; - uint8_t accessState; - int64_t totalStorage; - int64_t compStorage; - int64_t pointsWritten; - uint8_t syncStatus; - uint8_t reserved[15]; + int32_t vgId; + int64_t totalStorage; + int64_t compStorage; + int64_t pointsWritten; + uint8_t status; + uint8_t syncStatus; + uint8_t accessState; + uint8_t reserved[6]; } SVnodeLoad; typedef struct { @@ -582,14 +579,16 @@ typedef struct { } SVnodeStatisticInfo; typedef struct { + int32_t dnodeId; uint32_t moduleStatus; uint32_t createdTime; uint32_t numOfVnodes; - uint32_t reserved; } SDnodeState; typedef struct { uint32_t version; + int32_t dnodeId; + char dnodeName[TSDB_DNODE_NAME_LEN]; uint32_t privateIp; uint32_t publicIp; uint32_t lastReboot; // time stamp for last reboot @@ -603,14 +602,12 @@ typedef struct { } SDMStatusMsg; typedef struct { - int32_t code; - SDnodeState dnodeState; SRpcIpSet ipList; + SDnodeState dnodeState; SVnodeAccess vnodeAccess[]; } SDMStatusRsp; typedef struct { - int32_t vnode; SVnodeCfg cfg; SVnodeDesc vpeerDesc[TSDB_MAX_MPEERS]; } SMDCreateVnodeMsg; diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index 56a8fd054f..b7c68b5f80 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -32,15 +32,15 @@ SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode); void mgmtCreateVgroup(SQueuedMsg *pMsg); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); void mgmtUpdateVgroup(SVgObj *pVgroup); +void mgmtUpdateVgroupIp(SDnodeObj *pDnode); void mgmtSetVgroupIdPool(); SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb); void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable); void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable); - -SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode); -void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode, SRpcIpSet *ipSet, void *ahandle); +void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle); +void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle); SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup); SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip); diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 98a2e38728..ce7ac8cb39 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -299,62 +299,4 @@ static int mgmtDServerRetrieveAuth(char *user, char *spi, char *encrypt, char *s // mgmtCleanUpDnodeIntFp(); // } //} -// -//void mgmtProcessDnodeStatus(int8_t msgType, void *pCont, int32_t contLen, void *pConn, int32_t code) { -// SDMStatusMsg *pStatus = (SDMStatusMsg *)pCont; -// -// SDnodeObj *pObj = mgmtGetDnode(htonl(pStatus->privateIp)); -// if (pObj == NULL) { -// mError("dnode:%s not exist", taosIpStr(pObj->privateIp)); -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_DNODE_NOT_EXIST, NULL, 0); -// return; -// } -// -// pObj->lastReboot = htonl(pStatus->lastReboot); -// pObj->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes); -// pObj->openVnodes = htons(pStatus->openVnodes); -// pObj->numOfCores = htons(pStatus->numOfCores); -// pObj->diskAvailable = pStatus->diskAvailable; -// pObj->alternativeRole = pStatus->alternativeRole; -//// -//// if (mgmtProcessDnodeStatusFp) { -//// mgmtProcessDnodeStatusFp(pStatus, pObj, pConn); -//// return; -//// } -// -// pObj->status = TSDB_DN_STATUS_READY; -// -//// // wait vnode dropped -//// for (int32_t vnode = 0; vnode < pObj->numOfVnodes; ++vnode) { -//// SVnodeLoad *pVload = &(pObj->vload[vnode]); -//// if (pVload->dropStatus == TSDB_VN_DROP_STATUS_DROPPING) { -//// bool existInDnode = false; -//// for (int32_t j = 0; j < pObj->openVnodes; ++j) { -//// if (htonl(pStatus->load[j].vnode) == vnode) { -//// existInDnode = true; -//// break; -//// } -//// } -//// -//// if (!existInDnode) { -//// pVload->dropStatus = TSDB_VN_DROP_STATUS_READY; -//// pVload->status = TSDB_VN_STATUS_OFFLINE; -//// mgmtUpdateDnode(pObj); -//// mPrint("dnode:%s, vid:%d, drop finished", taosIpStr(pObj->privateIp), vnode); -//// taosTmrStart(mgmtMonitorDbDrop, 10000, NULL, tsMgmtTmr); -//// } -//// } else if (pVload->vgId == 0) { -//// /* -//// * In some cases, vnode information may be reported abnormally, recover it -//// */ -//// if (pVload->dropStatus != TSDB_VN_DROP_STATUS_READY || pVload->status != TSDB_VN_STATUS_OFFLINE) { -//// mPrint("dnode:%s, vid:%d, vgroup:%d status:%s dropStatus:%s, set it to avail status", -//// taosIpStr(pObj->privateIp), vnode, pVload->vgId, taosGetVnodeStatusStr(pVload->status), -//// taosGetVnodeDropStatusStr(pVload->dropStatus)); -//// pVload->dropStatus = TSDB_VN_DROP_STATUS_READY; -//// pVload->status = TSDB_VN_STATUS_OFFLINE; -//// mgmtUpdateDnode(pObj); -//// } -//// } -//// } -//} +// \ No newline at end of file diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index fcd9e58231..19ee53af0c 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -23,12 +23,14 @@ #include "mgmtDClient.h" #include "mgmtMnode.h" #include "mgmtShell.h" +#include "mgmtDServer.h" #include "mgmtUser.h" #include "mgmtVgroup.h" int32_t (*mgmtInitDnodesFp)() = NULL; void (*mgmtCleanUpDnodesFp)() = NULL; SDnodeObj *(*mgmtGetDnodeFp)(uint32_t ip) = NULL; +SDnodeObj *(*mgmtGetDnodeByIpFp)(int32_t dnodeId) = NULL; int32_t (*mgmtGetDnodesNumFp)() = NULL; int32_t (*mgmtUpdateDnodeFp)(SDnodeObj *pDnode) = NULL; void * (*mgmtGetNextDnodeFp)(SShowObj *pShow, SDnodeObj **pDnode) = NULL; @@ -45,6 +47,7 @@ static int32_t mgmtGetVnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg); static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; +static void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg); void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode) { int32_t maxVnodes = pDnode->numOfCores * tsNumOfVnodesPerCore; @@ -70,10 +73,9 @@ void mgmtCalcNumOfFreeVnodes(SDnodeObj *pDnode) { for (int32_t i = 0; i < pDnode->numOfVnodes; ++i) { SVnodeLoad *pVload = pDnode->vload + i; if (pVload->vgId != 0) { - mTrace("%d-dnode:%s, calc free vnodes, exist vnode:%d, vgroup:%d, state:%d %s, dropstate:%d %s, syncstatus:%d %s", - totalVnodes, taosIpStr(pDnode->privateIp), i, pVload->vgId, + mTrace("dnode:%d, calc free vnodes, vnode:%d, status:%d %s, syncstatus:%d %s", + pDnode->dnodeId, pVload->vgId, pVload->status, taosGetVnodeStatusStr(pVload->status), - pVload->dropStatus, taosGetVnodeDropStatusStr(pVload->dropStatus), pVload->syncStatus, taosGetVnodeSyncStatusStr(pVload->syncStatus)); totalVnodes++; } @@ -92,7 +94,7 @@ void mgmtSetDnodeVgid(SVnodeGid vnodeGid[], int32_t numOfVnodes, int32_t vgId) { if (pDnode) { SVnodeLoad *pVload = pDnode->vload + vnodeGid[i].vnode; memset(pVload, 0, sizeof(SVnodeLoad)); - pVload->vnode = vnodeGid[i].vnode; + //pVload->vnode = vnodeGid[i].vnode; pVload->vgId = vgId; mTrace("dnode:%s, vnode:%d add to vgroup:%d", taosIpStr(pDnode->privateIp), vnodeGid[i].vnode, pVload->vgId); mgmtCalcNumOfFreeVnodes(pDnode); @@ -118,7 +120,6 @@ void mgmtUnSetDnodeVgid(SVnodeGid vnodeGid[], int32_t numOfVnodes) { } } - bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { uint32_t status = pDnode->moduleStatus & (1 << moduleType); return status > 0; @@ -319,12 +320,6 @@ static int32_t mgmtGetVnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "vgid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - pShow->bytes[cols] = 12; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); @@ -400,10 +395,6 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi cols = 0; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(uint32_t *)pWrite = pVnode->vnode; - cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(uint32_t *)pWrite = pVnode->vgId; cols++; @@ -437,11 +428,12 @@ int32_t mgmtInitDnodes() { mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mgmtRetrieveVnodes); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp); - + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg); if (mgmtInitDnodesFp) { return mgmtInitDnodesFp(); } else { + tsDnodeObj.dnodeId = 1; tsDnodeObj.privateIp = inet_addr(tsPrivateIp);; tsDnodeObj.createdTime = taosGetTimestampMs(); tsDnodeObj.lastReboot = taosGetTimestampSec(); @@ -478,6 +470,16 @@ SDnodeObj *mgmtGetDnode(uint32_t ip) { } } +SDnodeObj *mgmtGetDnodeByIp(int32_t dnodeId) { + if (mgmtGetDnodeByIpFp) { + return mgmtGetDnodeByIpFp(dnodeId); + } + if (dnodeId != 0) { + return &tsDnodeObj; + } + return NULL; +} + int32_t mgmtGetDnodesNum() { if (mgmtGetDnodesNumFp) { return mgmtGetDnodesNumFp(); @@ -569,4 +571,130 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { mTrace("cfg vnode rsp is received"); -} \ No newline at end of file +} + +void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMStatusMsg *pStatus = rpcMsg->pCont; + pStatus->dnodeId = htonl(pStatus->dnodeId); + + SDnodeObj *pDnode = NULL; + if (pStatus->dnodeId == 0) { + pDnode = mgmtGetDnodeByIp(pStatus->privateIp); + if (pDnode == NULL) { + mTrace("dnode not created in cluster, privateIp:%s, name:%s, ", taosIpStr(htonl(pStatus->dnodeId)), pStatus->dnodeName); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST); + return; + } + } + + uint32_t version = htonl(pStatus->version); + if (version != tsVersion) { + mError("dnode:%d, status msg version:%d not equal with mnode:%d", pDnode->dnodeId, version, tsVersion); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_MSG_VERSION); + return ; + } + + uint32_t lastPrivateIp = htonl(pDnode->privateIp); + uint32_t lastPublicIp = htonl(pDnode->publicIp); + + pDnode->privateIp = htonl(pStatus->privateIp); + pDnode->publicIp = htonl(pStatus->publicIp); + pDnode->lastReboot = htonl(pStatus->lastReboot); + pDnode->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes); + pDnode->openVnodes = htons(pStatus->openVnodes); + pDnode->numOfCores = htons(pStatus->numOfCores); + pDnode->diskAvailable = pStatus->diskAvailable; + pDnode->alternativeRole = pStatus->alternativeRole; + + if (pStatus->dnodeId == 0) { + mTrace("dnode:%d, first access, privateIp:%s, name:%s, ", pDnode->dnodeId, taosIpStr(pStatus->dnodeId), pStatus->dnodeName); + mgmtSetDnodeMaxVnodes(pDnode); + mgmtUpdateDnode(pDnode); + } + + if (lastPrivateIp != pDnode->privateIp || lastPublicIp != pDnode->publicIp) { + mgmtUpdateVgroupIp(pDnode); + //mgmtUpdateMnodeIp(); + } + + for (int32_t j = 0; j < pDnode->openVnodes; ++j) { + pStatus->load[j].vgId = htonl(pStatus->load[j].vgId); + pStatus->load[j].totalStorage = htobe64(pStatus->load[j].totalStorage); + pStatus->load[j].compStorage = htobe64(pStatus->load[j].compStorage); + pStatus->load[j].pointsWritten = htobe64(pStatus->load[j].pointsWritten); + + bool existInMnode = false; + for (int32_t vnode = 0; vnode < pDnode->numOfVnodes; ++vnode) { + SVnodeLoad *pVload = &(pDnode->vload[vnode]); + if (pVload->vgId == pStatus->load[j].vgId) { + existInMnode = true; + } + } + + if (!existInMnode) { + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); + mPrint("dnode:%d, vnode:%d not exist in mnode, drop it", pDnode->dnodeId, pStatus->load[j].vgId); + mgmtSendDropVnodeMsg(pStatus->load[j].vgId, &ipSet, NULL); + } + } + + for (int32_t vnode = 0; vnode < pDnode->numOfVnodes; ++vnode) { + SVnodeLoad *pVload = &(pDnode->vload[vnode]); + + bool existInDnode = false; + for (int32_t j = 0; j < pDnode->openVnodes; ++j) { + if (htonl(pStatus->load[j].vgId) == pVload->vgId) { + existInDnode = true; + break; + } + } + + if (!existInDnode) { + mPrint("dnode:%d, vnode:%d not exist in dnode, create it", pDnode->dnodeId, pVload->vgId); + SVgObj *pVgroup = mgmtGetVgroup(pVload->vgId); + if (pVgroup != NULL) { + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); + mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL); + } + } + } + + if (pDnode->status != TSDB_DN_STATUS_READY) { + mTrace("dnode:%d, from offline to online", pDnode->dnodeId); + pDnode->status = TSDB_DN_STATUS_READY; + //TODO: + //mgmtStartBalanceTimer(200); + } + + int32_t contLen = sizeof(SDMStatusRsp) + TSDB_MAX_VNODES * sizeof(SVnodeAccess); + SDMStatusRsp *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; + } + + pRsp->ipList = *pSdbIpList; + pRsp->ipList.port = htons(pRsp->ipList.port); + for (int i = 0; i < pRsp->ipList.numOfIps; ++i) { + pRsp->ipList.ip[i] = htonl(pRsp->ipList.ip[i]); + } + + pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); + pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); + pRsp->dnodeState.createdTime = htonl(pDnode->createdTime / 1000); + pRsp->dnodeState.numOfVnodes = 0; + + contLen = sizeof(SDMStatusRsp); + + //TODO: set vnode access + + SRpcMsg rpcRsp = { + .code = TSDB_CODE_SUCCESS, + .pCont = pStatus, + .contLen = contLen + }; + + rpcSendResponse(&rpcRsp); +} diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 637d0a1107..f53ca3f95f 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -19,4 +19,5 @@ bool mgmtCheckRedirect(void *handle) { return false; -} \ No newline at end of file +} + diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 223e67ebfb..e87c964496 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -392,7 +392,7 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { } if (pConnectMsg->db[0]) { - char dbName[TSDB_TABLE_ID_LEN] = {0}; + char dbName[TSDB_TABLE_ID_LEN * 3] = {0}; sprintf(dbName, "%x%s%s", pAcct->acctId, TS_PATH_DELIMITER, pConnectMsg->db); SDbObj *pDb = mgmtGetDb(dbName); if (pDb == NULL) { diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index c2f3e19951..7faaed1d80 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -316,9 +316,9 @@ char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) { } SVnodeLoad *vload = pDnode->vload + pVnode->vnode; - if (vload->vgId != pVgroup->vgId || vload->vnode != pVnode->vnode) { - mError("dnode:%s, vgroup:%d, vnode:%d not same with dnode vgroup:%d vnode:%d", - taosIpStr(pVnode->ip), pVgroup->vgId, pVnode->vnode, vload->vgId, vload->vnode); + if (vload->vgId != pVgroup->vgId) { + mError("dnode:%s, vgroup:%d, not same with dnode vgroup:%d", + taosIpStr(pVnode->ip), pVgroup->vgId, vload->vgId); return "null"; } @@ -489,15 +489,14 @@ void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { taosFreeId(pVgroup->idPool, pTable->sid); } -SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode) { +SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) return NULL; SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); if (pVnode == NULL) return NULL; - pVnode->vnode = htonl(vnode); - pVnode->cfg = pDb->cfg; + pVnode->cfg = pDb->cfg; SVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); @@ -517,7 +516,6 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode) { for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { vpeerDesc[j].vgId = htonl(pVgroup->vgId); vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].ip); - vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode); } return pVnode; @@ -559,9 +557,9 @@ SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip) { return ipSet; } -void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode, SRpcIpSet *ipSet, void *ahandle) { - mTrace("vgroup:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, vnode, ahandle); - SMDCreateVnodeMsg *pCreate = mgmtBuildCreateVnodeMsg(pVgroup, vnode); +void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { + mTrace("vgroup:%d, send create msg, ahandle:%p", pVgroup->vgId, ahandle); + SMDCreateVnodeMsg *pCreate = mgmtBuildCreateVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .handle = ahandle, .pCont = pCreate, @@ -576,7 +574,7 @@ void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) { mTrace("vgroup:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].ip); - mgmtSendCreateVnodeMsg(pVgroup, pVgroup->vnodeGid[i].vnode, &ipSet, ahandle); + mgmtSendCreateVnodeMsg(pVgroup, &ipSet, ahandle); } } @@ -615,17 +613,17 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { free(queueMsg); } -static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(SVgObj *pVgroup) { +static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) { SMDDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SMDDropVnodeMsg)); if (pDrop == NULL) return NULL; - pDrop->vgId = htonl(pVgroup->vgId); + pDrop->vgId = htonl(vgId); return pDrop; } -static void mgmtSendDropVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { - mTrace("vgroup:%d, send drop vnode msg, ahandle:%p", pVgroup->vgId, ahandle); - SMDDropVnodeMsg *pDrop = mgmtBuildDropVnodeMsg(pVgroup); +void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { + mTrace("vgroup:%d, send drop vnode msg, ahandle:%p", vgId, ahandle); + SMDDropVnodeMsg *pDrop = mgmtBuildDropVnodeMsg(vgId); SRpcMsg rpcMsg = { .handle = ahandle, .pCont = pDrop, @@ -640,7 +638,7 @@ static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { mTrace("vgroup:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].ip); - mgmtSendDropVnodeMsg(pVgroup, &ipSet, ahandle); + mgmtSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle); } } @@ -675,4 +673,25 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { mgmtAddToShellQueue(newMsg); free(queueMsg); -} \ No newline at end of file +} + +void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { + void * pNode = NULL; + SVgObj *pVgroup = NULL; + while (1) { + pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); + if (pVgroup == NULL) break; + + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SVnodeGid *vnodeGid = pVgroup->vnodeGid + i; + if (vnodeGid->dnodeId == pDnode->dnodeId) { + mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s", + pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->ip), + pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); + vnodeGid->publicIp = pDnode->publicIp; + vnodeGid->ip = pDnode->privateIp; + sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); + } + } + } +} diff --git a/src/util/inc/ihash.h b/src/util/inc/ihash.h index 1d7a8f7930..f283abe737 100644 --- a/src/util/inc/ihash.h +++ b/src/util/inc/ihash.h @@ -36,7 +36,7 @@ int32_t taosHashInt(void *handle, uint64_t key); void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)); -char *taosVisitIntHashWithFp(void *handle, int (*fp)(char *)); +void taosVisitIntHashWithFp(void *handle, void (*fp)(char *, void *), void *param); int32_t taosGetIntHashSize(void *handle); diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c index a61ce6654f..6b58d8ef31 100644 --- a/src/util/src/ihash.c +++ b/src/util/src/ihash.c @@ -187,7 +187,6 @@ void taosCleanUpIntHash(void *handle) { free(pObj); } - void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)) { IHashObj * pObj; IHashNode *pNode, *pNext; @@ -202,7 +201,7 @@ void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)) { pNode = pObj->hashList[i]; while (pNode) { pNext = pNode->next; - if (fp != NULL) fp(pNode->data); + if (fp != NULL) (*fp)(pNode->data); free(pNode); pNode = pNext; } @@ -219,7 +218,7 @@ void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)) { free(pObj); } -char *taosVisitIntHashWithFp(void *handle, int (*fp)(char *)) { +void taosVisitIntHashWithFp(void *handle, int (*fp)(char *, void *), void *param) { IHashObj * pObj; IHashNode *pNode, *pNext; char * pData = NULL; @@ -234,21 +233,13 @@ char *taosVisitIntHashWithFp(void *handle, int (*fp)(char *)) { pNode = pObj->hashList[i]; while (pNode) { pNext = pNode->next; - int flag = fp(pNode->data); - if (flag) { - pData = pNode->data; - goto VisitEnd; - } - + (*fp)(pNode->data, param); pNode = pNext; } } } -VisitEnd: - pthread_mutex_unlock(&pObj->mutex); - return pData; } int32_t taosGetIntHashSize(void *handle) { diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c index bb7fc3f0b1..39704464e8 100644 --- a/src/util/src/tstatus.c +++ b/src/util/src/tstatus.c @@ -18,13 +18,13 @@ char* taosGetVgroupStatusStr(int32_t vgroupStatus) { switch (vgroupStatus) { - case TSDB_VG_STATUS_READY: return tstrerror(vgroupStatus); - case TSDB_VG_STATUS_IN_PROGRESS: return tstrerror(vgroupStatus); - case TSDB_VG_STATUS_NO_DISK_PERMISSIONS: return tstrerror(vgroupStatus); - case TSDB_VG_STATUS_SERVER_NO_PACE: return tstrerror(vgroupStatus); - case TSDB_VG_STATUS_SERV_OUT_OF_MEMORY: return tstrerror(vgroupStatus); - case TSDB_VG_STATUS_INIT_FAILED: return tstrerror(vgroupStatus); - case TSDB_VG_STATUS_FULL: return tstrerror(vgroupStatus); + case TSDB_VG_STATUS_READY: return (char*)tstrerror(vgroupStatus); + case TSDB_VG_STATUS_IN_PROGRESS: return (char*)tstrerror(vgroupStatus); + case TSDB_VG_STATUS_NO_DISK_PERMISSIONS: return (char*)tstrerror(vgroupStatus); + case TSDB_VG_STATUS_SERVER_NO_PACE: return (char*)tstrerror(vgroupStatus); + case TSDB_VG_STATUS_SERV_OUT_OF_MEMORY: return (char*)tstrerror(vgroupStatus); + case TSDB_VG_STATUS_INIT_FAILED: return (char*)tstrerror(vgroupStatus); + case TSDB_VG_STATUS_FULL: return (char*)tstrerror(vgroupStatus); default: return "undefined"; } } diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index 9b7215d7da..f6e97aefb0 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -35,7 +35,6 @@ extern "C" { // --------- TSDB REPOSITORY CONFIGURATION DEFINITION typedef struct { int8_t precision; - int32_t vgId; int32_t tsdbId; int32_t maxTables; // maximum number of tables this repository can have int32_t daysPerFile; // day per file sharding policy -- GitLab