dmMgmt.c 10.8 KB
Newer Older
S
shm  
Shengliang Guan 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

S
shm  
Shengliang Guan 已提交
16
#define _DEFAULT_SOURCE
S
Shengliang Guan 已提交
17
#include "dmMgmt.h"
S
Shengliang Guan 已提交
18

S
Shengliang Guan 已提交
19 20
static bool dmIsNodeDeployedFp(SDnode *pDnode, EDndNodeType ntype) { return pDnode->wrappers[ntype].required; }

S
Shengliang Guan 已提交
21
static int32_t dmInitVars(SDnode *pDnode, const SDnodeOpt *pOption) {
S
Shengliang Guan 已提交
22 23 24 25 26 27 28 29 30 31 32 33
  pDnode->input.dnodeId = 0;
  pDnode->input.clusterId = 0;
  pDnode->input.localEp = strdup(pOption->localEp);
  pDnode->input.localFqdn = strdup(pOption->localFqdn);
  pDnode->input.firstEp = strdup(pOption->firstEp);
  pDnode->input.secondEp = strdup(pOption->secondEp);
  pDnode->input.serverPort = pOption->serverPort;
  pDnode->input.supportVnodes = pOption->numOfSupportVnodes;
  pDnode->input.numOfDisks = pOption->numOfDisks;
  pDnode->input.disks = pOption->disks;
  pDnode->input.dataDir = strdup(pOption->dataDir);
  pDnode->input.pDnode = pDnode;
S
Shengliang Guan 已提交
34 35 36
  pDnode->input.processCreateNodeFp = dmProcessCreateNodeReq;
  pDnode->input.processDropNodeFp = dmProcessDropNodeReq;
  pDnode->input.isNodeDeployedFp = dmIsNodeDeployedFp;
S
shm  
Shengliang Guan 已提交
37

S
Shengliang Guan 已提交
38 39
  if (pDnode->input.dataDir == NULL || pDnode->input.localEp == NULL || pDnode->input.localFqdn == NULL ||
      pDnode->input.firstEp == NULL || pDnode->input.secondEp == NULL) {
S
shm  
Shengliang Guan 已提交
40 41 42
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return -1;
  }
S
shm  
Shengliang Guan 已提交
43

S
Shengliang Guan 已提交
44
  pDnode->ntype = pOption->ntype;
S
Shengliang Guan 已提交
45
  if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_END) {
S
Shengliang Guan 已提交
46 47
    pDnode->lockfile = dmCheckRunning(pOption->dataDir);
    if (pDnode->lockfile == NULL) {
S
shm  
Shengliang Guan 已提交
48 49
      return -1;
    }
S
shm  
Shengliang Guan 已提交
50 51
  }

52
  taosThreadMutexInit(&pDnode->mutex, NULL);
S
shm  
Shengliang Guan 已提交
53 54 55
  return 0;
}

S
Shengliang Guan 已提交
56
static void dmClearVars(SDnode *pDnode) {
S
Shengliang Guan 已提交
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
  for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
    SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
    taosMemoryFreeClear(pWrapper->path);
  }
  if (pDnode->lockfile != NULL) {
    taosUnLockFile(pDnode->lockfile);
    taosCloseFile(&pDnode->lockfile);
    pDnode->lockfile = NULL;
  }

  taosMemoryFreeClear(pDnode->input.localEp);
  taosMemoryFreeClear(pDnode->input.localFqdn);
  taosMemoryFreeClear(pDnode->input.firstEp);
  taosMemoryFreeClear(pDnode->input.secondEp);
  taosMemoryFreeClear(pDnode->input.dataDir);
S
Shengliang Guan 已提交
72 73 74 75

  taosThreadMutexDestroy(&pDnode->mutex);
  memset(&pDnode->mutex, 0, sizeof(pDnode->mutex));
  taosMemoryFree(pDnode);
S
shm  
Shengliang Guan 已提交
76
  dDebug("dnode memory is cleared, data:%p", pDnode);
S
shm  
Shengliang Guan 已提交
77 78
}

S
Shengliang Guan 已提交
79
static bool dmRequireNode(SMgmtWrapper *pWrapper) {
S
Shengliang Guan 已提交
80 81 82 83
  SMgmtInputOpt *pInput = &pWrapper->pDnode->input;
  pInput->name = pWrapper->name;
  pInput->path = pWrapper->path;

S
Shengliang Guan 已提交
84
  bool    required = false;
S
Shengliang Guan 已提交
85
  int32_t code = (*pWrapper->func.requiredFp)(pInput, &required);
S
Shengliang Guan 已提交
86 87 88 89 90 91
  if (!required) {
    dDebug("node:%s, does not require startup", pWrapper->name);
  }
  return required;
}

S
Shengliang Guan 已提交
92
SDnode *dmCreate(const SDnodeOpt *pOption) {
S
Shengliang Guan 已提交
93
  dInfo("start to create dnode");
S
shm  
Shengliang Guan 已提交
94
  int32_t code = -1;
S
Shengliang Guan 已提交
95
  char    path[PATH_MAX + 100] = {0};
S
shm  
Shengliang Guan 已提交
96 97
  SDnode *pDnode = NULL;

wafwerar's avatar
wafwerar 已提交
98
  pDnode = taosMemoryCalloc(1, sizeof(SDnode));
S
shm  
Shengliang Guan 已提交
99 100 101 102 103
  if (pDnode == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    goto _OVER;
  }

S
Shengliang Guan 已提交
104 105 106 107
  if (dmInitVars(pDnode, pOption) != 0) {
    goto _OVER;
  }

S
Shengliang Guan 已提交
108
  dmSetStatus(pDnode, DND_STAT_INIT);
S
Shengliang Guan 已提交
109 110 111 112 113 114
  pDnode->wrappers[DNODE].func = dmGetMgmtFunc();
  pDnode->wrappers[MNODE].func = mmGetMgmtFunc();
  pDnode->wrappers[VNODE].func = vmGetMgmtFunc();
  pDnode->wrappers[QNODE].func = qmGetMgmtFunc();
  pDnode->wrappers[SNODE].func = smGetMgmtFunc();
  pDnode->wrappers[BNODE].func = bmGetMgmtFunc();
S
shm  
Shengliang Guan 已提交
115

S
Shengliang 已提交
116 117
  for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
    SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
S
shm  
Shengliang Guan 已提交
118
    pWrapper->pDnode = pDnode;
S
Shengliang Guan 已提交
119 120
    pWrapper->name = dmNodeName(ntype);
    pWrapper->procShm.id = -1;
S
Shengliang 已提交
121
    pWrapper->nodeType = ntype;
S
Shengliang Guan 已提交
122 123 124
    pWrapper->procType = DND_PROC_SINGLE;
    taosInitRWLatch(&pWrapper->latch);

S
Shengliang Guan 已提交
125 126
    snprintf(path, sizeof(path), "%s%s%s", pOption->dataDir, TD_DIRSEP, pWrapper->name);
    pWrapper->path = strdup(path);
S
shm  
Shengliang Guan 已提交
127 128 129 130 131
    if (pWrapper->path == NULL) {
      terrno = TSDB_CODE_OUT_OF_MEMORY;
      goto _OVER;
    }

S
Shengliang Guan 已提交
132
    if (ntype != DNODE && dmReadShmFile(pWrapper->path, pWrapper->name, pDnode->ntype, &pWrapper->procShm) != 0) {
S
Shengliang Guan 已提交
133 134 135
      dError("node:%s, failed to read shm file since %s", pWrapper->name, terrstr());
      goto _OVER;
    }
S
Shengliang Guan 已提交
136 137

    pWrapper->required = dmRequireNode(pWrapper);
S
shm  
Shengliang Guan 已提交
138 139
  }

S
Shengliang Guan 已提交
140
  if (dmInitMsgHandle(pDnode) != 0) {
S
Shengliang Guan 已提交
141
    dError("failed to init msg handles since %s", terrstr());
S
shm  
Shengliang Guan 已提交
142 143 144
    goto _OVER;
  }

S
Shengliang Guan 已提交
145 146 147 148
  if (dmInitClient(pDnode) != 0) {
    goto _OVER;
  }

S
shm  
Shengliang Guan 已提交
149
  dInfo("dnode is created, data:%p", pDnode);
S
shm  
Shengliang Guan 已提交
150 151 152
  code = 0;

_OVER:
S
Shengliang Guan 已提交
153
  if (code != 0 && pDnode != NULL) {
S
Shengliang Guan 已提交
154
    dmClearVars(pDnode);
155
    pDnode = NULL;
S
shm  
Shengliang Guan 已提交
156
    dError("failed to create dnode since %s", terrstr());
S
shm  
Shengliang Guan 已提交
157 158 159 160 161
  }

  return pDnode;
}

S
Shengliang Guan 已提交
162
void dmClose(SDnode *pDnode) {
S
shm  
Shengliang Guan 已提交
163
  if (pDnode == NULL) return;
S
Shengliang Guan 已提交
164 165 166 167

  dmCleanupClient(pDnode);
  dmCleanupServer(pDnode);

S
Shengliang Guan 已提交
168
  dmClearVars(pDnode);
S
shm  
Shengliang Guan 已提交
169
  dInfo("dnode is closed, data:%p", pDnode);
S
shm  
Shengliang Guan 已提交
170
}
S
Shengliang Guan 已提交
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237

void dmSetStatus(SDnode *pDnode, EDndRunStatus status) {
  if (pDnode->status != status) {
    dDebug("dnode status set from %s to %s", dmStatStr(pDnode->status), dmStatStr(status));
    pDnode->status = status;
  }
}

void dmSetEvent(SDnode *pDnode, EDndEvent event) {
  if (event == DND_EVENT_STOP) {
    pDnode->event = event;
  }
}

SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) {
  SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
  SMgmtWrapper *pRetWrapper = pWrapper;

  taosRLockLatch(&pWrapper->latch);
  if (pWrapper->deployed) {
    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
    dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount);
  } else {
    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
    pRetWrapper = NULL;
  }
  taosRUnLockLatch(&pWrapper->latch);

  return pRetWrapper;
}

int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) {
  int32_t code = 0;

  taosRLockLatch(&pWrapper->latch);
  if (pWrapper->deployed || (pWrapper->procType == DND_PROC_PARENT && pWrapper->required)) {
    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
    dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount);
  } else {
    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
    code = -1;
  }
  taosRUnLockLatch(&pWrapper->latch);

  return code;
}

void dmReleaseWrapper(SMgmtWrapper *pWrapper) {
  if (pWrapper == NULL) return;

  taosRLockLatch(&pWrapper->latch);
  int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1);
  taosRUnLockLatch(&pWrapper->latch);
  dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount);
}

void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
  SStartupInfo *pStartup = &pDnode->startup;
  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
  dInfo("step:%s, %s", pStartup->name, pStartup->desc);
}

void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc) {
  dmReportStartup(pWrapper->pDnode, pName, pDesc);
}

238
static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus) {
S
Shengliang Guan 已提交
239
  SDnodeMgmt *pMgmt = pDnode->wrappers[DNODE].pMgmt;
S
Shengliang Guan 已提交
240 241 242 243 244
  pStatus->details[0] = 0;

  if (pDnode->status == DND_STAT_INIT) {
    pStatus->statusCode = TSDB_SRV_STATUS_NETWORK_OK;
    snprintf(pStatus->details, sizeof(pStatus->details), "%s: %s", pDnode->startup.name, pDnode->startup.desc);
245
  } else if (pDnode->status == DND_STAT_STOPPED) {
S
Shengliang Guan 已提交
246
    pStatus->statusCode = TSDB_SRV_STATUS_EXTING;
247 248
  } else {
    pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK;
S
Shengliang Guan 已提交
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
  }
}

void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) {
  dDebug("net test req is received");
  SRpcMsg rsp = {.handle = pReq->handle, .refId = pReq->refId, .ahandle = pReq->ahandle, .code = 0};
  rsp.pCont = rpcMallocCont(pReq->contLen);
  if (rsp.pCont == NULL) {
    rsp.code = TSDB_CODE_OUT_OF_MEMORY;
  } else {
    rsp.contLen = pReq->contLen;
  }
  rpcSendResponse(&rsp);
  rpcFreeCont(pReq->pCont);
}

265 266
void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pReq) {
  dDebug("server startup status req is received");
S
Shengliang Guan 已提交
267 268

  SServerStatusRsp statusRsp = {0};
269
  dmGetServerStartupStatus(pDnode, &statusRsp);
S
Shengliang Guan 已提交
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310

  SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle, .refId = pReq->refId};
  int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
  if (rspLen < 0) {
    rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
    goto _OVER;
  }

  void *pRsp = rpcMallocCont(rspLen);
  if (pRsp == NULL) {
    rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
    goto _OVER;
  }

  tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
  rspMsg.pCont = pRsp;
  rspMsg.contLen = rspLen;

_OVER:
  rpcSendResponse(&rspMsg);
  rpcFreeCont(pReq->pCont);
}

int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) {
  SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
  if (pWrapper != NULL) {
    dmReleaseWrapper(pWrapper);
    terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
    dError("failed to create node since %s", terrstr());
    return -1;
  }

  taosThreadMutexLock(&pDnode->mutex);
  pWrapper = &pDnode->wrappers[ntype];

  if (taosMkDir(pWrapper->path) != 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
    return -1;
  }

S
Shengliang Guan 已提交
311 312 313 314 315 316
  SMgmtInputOpt *pInput = &pWrapper->pDnode->input;
  pInput->name = pWrapper->name;
  pInput->path = pWrapper->path;
  pInput->msgCb = dmGetMsgcb(pWrapper);

  int32_t code = (*pWrapper->func.createFp)(pInput, pMsg);
S
Shengliang Guan 已提交
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
  if (code != 0) {
    dError("node:%s, failed to create since %s", pWrapper->name, terrstr());
  } else {
    dDebug("node:%s, has been created", pWrapper->name);
    (void)dmOpenNode(pWrapper);
    pWrapper->required = true;
    pWrapper->deployed = true;
    pWrapper->procType = pDnode->ptype;
  }

  taosThreadMutexUnlock(&pDnode->mutex);
  return code;
}

int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) {
  SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
  if (pWrapper == NULL) {
    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
    dError("failed to drop node since %s", terrstr());
    return -1;
  }

  taosThreadMutexLock(&pDnode->mutex);

S
Shengliang Guan 已提交
341
  int32_t code = (*pWrapper->func.dropFp)(pWrapper->pMgmt, pMsg);
S
Shengliang Guan 已提交
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
  if (code != 0) {
    dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
  } else {
    dDebug("node:%s, has been dropped", pWrapper->name);
    pWrapper->required = false;
    pWrapper->deployed = false;
  }

  dmReleaseWrapper(pWrapper);

  if (code == 0) {
    dmCloseNode(pWrapper);
    taosRemoveDir(pWrapper->path);
  }
  taosThreadMutexUnlock(&pDnode->mutex);
  return code;
}