dndEnv.c 9.1 KB
Newer Older
S
Shengliang Guan 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#define _DEFAULT_SOURCE
S
Shengliang Guan 已提交
17
#include "dndBnode.h"
S
Shengliang Guan 已提交
18
#include "dndMgmt.h"
S
Shengliang Guan 已提交
19
#include "dndMnode.h"
S
Shengliang Guan 已提交
20 21
#include "dndQnode.h"
#include "dndSnode.h"
S
Shengliang Guan 已提交
22 23
#include "dndTransport.h"
#include "dndVnodes.h"
S
Shengliang Guan 已提交
24
#include "sync.h"
S
Shengliang Guan 已提交
25
#include "tfs.h"
S
Shengliang Guan 已提交
26
#include "wal.h"
S
Shengliang Guan 已提交
27

S
Shengliang Guan 已提交
28 29
static SDnodeEnv dndEnv = {0};

S
Shengliang Guan 已提交
30
EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; }
S
Shengliang Guan 已提交
31

S
Shengliang Guan 已提交
32
void dndSetStat(SDnode *pDnode, EStat stat) {
S
Shengliang Guan 已提交
33
  dDebug("dnode status set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat));
S
Shengliang Guan 已提交
34 35 36
  pDnode->stat = stat;
}

S
Shengliang Guan 已提交
37
const char *dndStatStr(EStat stat) {
S
Shengliang Guan 已提交
38
  switch (stat) {
S
Shengliang Guan 已提交
39
    case DND_STAT_INIT:
S
Shengliang Guan 已提交
40
      return "init";
S
Shengliang Guan 已提交
41
    case DND_STAT_RUNNING:
S
Shengliang Guan 已提交
42
      return "running";
S
Shengliang Guan 已提交
43
    case DND_STAT_STOPPED:
S
Shengliang Guan 已提交
44 45 46 47 48 49
      return "stopped";
    default:
      return "unknown";
  }
}

S
Shengliang Guan 已提交
50
void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) {
S
Shengliang Guan 已提交
51
  SStartupReq *pStartup = &pDnode->startup;
S
Shengliang Guan 已提交
52 53
  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
S
Shengliang Guan 已提交
54 55 56
  pStartup->finished = 0;
}

S
Shengliang Guan 已提交
57 58
void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
S
Shengliang Guan 已提交
59
  pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
S
Shengliang Guan 已提交
60 61
}

S
Shengliang Guan 已提交
62
static FileFd dndCheckRunning(char *dataDir) {
S
Shengliang Guan 已提交
63 64 65 66 67 68
  char filepath[PATH_MAX] = {0};
  snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);

  FileFd fd = taosOpenFileCreateWriteTrunc(filepath);
  if (fd < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
69
    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
S
Shengliang Guan 已提交
70 71 72 73 74 75
    return -1;
  }

  int32_t ret = taosLockFile(fd);
  if (ret != 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
76
    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
S
Shengliang Guan 已提交
77 78 79 80
    taosCloseFile(fd);
    return -1;
  }

S
Shengliang Guan 已提交
81
  return fd;
S
Shengliang Guan 已提交
82 83
}

S
Shengliang Guan 已提交
84 85
static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) {
  pDnode->lockFd = dndCheckRunning(pCfg->dataDir);
S
Shengliang Guan 已提交
86
  if (pDnode->lockFd < 0) {
S
Shengliang Guan 已提交
87 88 89 90
    return -1;
  }

  char path[PATH_MAX + 100];
S
Shengliang Guan 已提交
91
  snprintf(path, sizeof(path), "%s%smnode", pCfg->dataDir, TD_DIRSEP);
92
  pDnode->dir.mnode = tstrdup(path);
S
Shengliang Guan 已提交
93
  snprintf(path, sizeof(path), "%s%svnode", pCfg->dataDir, TD_DIRSEP);
94
  pDnode->dir.vnodes = tstrdup(path);
S
Shengliang Guan 已提交
95
  snprintf(path, sizeof(path), "%s%sdnode", pCfg->dataDir, TD_DIRSEP);
96
  pDnode->dir.dnode = tstrdup(path);
S
Shengliang Guan 已提交
97
  snprintf(path, sizeof(path), "%s%ssnode", pCfg->dataDir, TD_DIRSEP);
S
Shengliang Guan 已提交
98
  pDnode->dir.snode = tstrdup(path);
S
Shengliang Guan 已提交
99
  snprintf(path, sizeof(path), "%s%sbnode", pCfg->dataDir, TD_DIRSEP);
S
Shengliang Guan 已提交
100
  pDnode->dir.bnode = tstrdup(path);
S
Shengliang Guan 已提交
101

S
Shengliang Guan 已提交
102 103
  if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL ||
      pDnode->dir.snode == NULL || pDnode->dir.bnode == NULL) {
S
Shengliang Guan 已提交
104 105 106 107 108
    dError("failed to malloc dir object");
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return -1;
  }

109
  if (taosMkDir(pDnode->dir.dnode) != 0) {
S
Shengliang Guan 已提交
110 111 112 113 114
    dError("failed to create dir:%s since %s", pDnode->dir.dnode, strerror(errno));
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

115
  if (taosMkDir(pDnode->dir.mnode) != 0) {
S
Shengliang Guan 已提交
116 117 118 119 120
    dError("failed to create dir:%s since %s", pDnode->dir.mnode, strerror(errno));
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

121
  if (taosMkDir(pDnode->dir.vnodes) != 0) {
S
Shengliang Guan 已提交
122 123 124 125 126
    dError("failed to create dir:%s since %s", pDnode->dir.vnodes, strerror(errno));
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

S
Shengliang Guan 已提交
127 128 129 130 131 132 133 134 135 136 137 138
  if (taosMkDir(pDnode->dir.snode) != 0) {
    dError("failed to create dir:%s since %s", pDnode->dir.snode, strerror(errno));
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  if (taosMkDir(pDnode->dir.bnode) != 0) {
    dError("failed to create dir:%s since %s", pDnode->dir.bnode, strerror(errno));
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

S
Shengliang Guan 已提交
139 140
  memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg));
  memcpy(&pDnode->env, &dndEnv.cfg, sizeof(SDnodeEnvCfg));
S
Shengliang Guan 已提交
141 142 143
  return 0;
}

S
Shengliang Guan 已提交
144
static void dndCloseImp(SDnode *pDnode) {
S
Shengliang Guan 已提交
145 146 147 148 149
  tfree(pDnode->dir.mnode);
  tfree(pDnode->dir.vnodes);
  tfree(pDnode->dir.dnode);
  tfree(pDnode->dir.snode);
  tfree(pDnode->dir.bnode);
S
Shengliang Guan 已提交
150

S
Shengliang Guan 已提交
151 152 153 154 155
  if (pDnode->lockFd >= 0) {
    taosUnLockFile(pDnode->lockFd);
    taosCloseFile(pDnode->lockFd);
    pDnode->lockFd = 0;
  }
S
Shengliang Guan 已提交
156 157
}

S
Shengliang Guan 已提交
158 159
SDnode *dndCreate(SDnodeObjCfg *pCfg) {
  dInfo("start to create dnode object");
S
Shengliang Guan 已提交
160

161
  SDnode *pDnode = calloc(1, sizeof(SDnode));
S
Shengliang Guan 已提交
162 163
  if (pDnode == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
S
Shengliang Guan 已提交
164
    dError("failed to create dnode object since %s", terrstr());
S
Shengliang Guan 已提交
165 166 167
    return NULL;
  }

S
Shengliang Guan 已提交
168
  dndSetStat(pDnode, DND_STAT_INIT);
S
Shengliang Guan 已提交
169

S
Shengliang Guan 已提交
170 171 172
  if (dndCreateImp(pDnode, pCfg) != 0) {
    dError("failed to init dnode dir since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
173 174 175
    return NULL;
  }

S
Shengliang Guan 已提交
176 177
  SDiskCfg dCfg = {0};
  tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN);
S
Shengliang Guan 已提交
178 179
  dCfg.level = 0;
  dCfg.primary = 1;
S
Shengliang Guan 已提交
180 181
  pDnode->pTfs = tfsOpen(&dCfg, 1);
  if (pDnode->pTfs == NULL) {
S
Shengliang Guan 已提交
182 183
    dError("failed to init tfs since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
184 185 186
    return NULL;
  }

S
Shengliang Guan 已提交
187
  if (dndInitMgmt(pDnode) != 0) {
S
Shengliang Guan 已提交
188 189
    dError("failed to init mgmt since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
190 191 192
    return NULL;
  }

S
Shengliang Guan 已提交
193
  if (dndInitVnodes(pDnode) != 0) {
S
Shengliang Guan 已提交
194 195
    dError("failed to init vnodes since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
196 197 198
    return NULL;
  }

S
Shengliang Guan 已提交
199
  if (dndInitQnode(pDnode) != 0) {
S
Shengliang Guan 已提交
200 201
    dError("failed to init qnode since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
202 203 204 205
    return NULL;
  }

  if (dndInitSnode(pDnode) != 0) {
S
Shengliang Guan 已提交
206 207
    dError("failed to init snode since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
208 209 210 211
    return NULL;
  }

  if (dndInitBnode(pDnode) != 0) {
S
Shengliang Guan 已提交
212 213
    dError("failed to init bnode since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
214 215 216
    return NULL;
  }

S
Shengliang Guan 已提交
217
  if (dndInitMnode(pDnode) != 0) {
S
Shengliang Guan 已提交
218 219
    dError("failed to init mnode since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
220 221 222
    return NULL;
  }

S
Shengliang Guan 已提交
223
  if (dndInitTrans(pDnode) != 0) {
S
Shengliang Guan 已提交
224 225
    dError("failed to init transport since %s", terrstr());
    dndClose(pDnode);
S
Shengliang Guan 已提交
226 227 228
    return NULL;
  }

S
Shengliang Guan 已提交
229
  dndSetStat(pDnode, DND_STAT_RUNNING);
S
Shengliang Guan 已提交
230
  dndSendStatusReq(pDnode);
S
Shengliang Guan 已提交
231
  dndReportStartup(pDnode, "TDengine", "initialized successfully");
S
Shengliang Guan 已提交
232
  dInfo("dnode object is created, data:%p", pDnode);
S
Shengliang Guan 已提交
233

234
  return pDnode;
S
Shengliang Guan 已提交
235 236
}

S
Shengliang Guan 已提交
237
void dndClose(SDnode *pDnode) {
S
Shengliang Guan 已提交
238 239
  if (pDnode == NULL) return;

S
Shengliang Guan 已提交
240
  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
S
Shengliang Guan 已提交
241
    dError("dnode is shutting down, data:%p", pDnode);
S
Shengliang Guan 已提交
242 243 244
    return;
  }

S
Shengliang Guan 已提交
245
  dInfo("start to close dnode, data:%p", pDnode);
S
Shengliang Guan 已提交
246 247
  dndSetStat(pDnode, DND_STAT_STOPPED);
  dndCleanupTrans(pDnode);
S
Shengliang Guan 已提交
248
  dndStopMgmt(pDnode);
S
Shengliang Guan 已提交
249
  dndCleanupMnode(pDnode);
S
Shengliang Guan 已提交
250 251 252
  dndCleanupBnode(pDnode);
  dndCleanupSnode(pDnode);
  dndCleanupQnode(pDnode);
S
Shengliang Guan 已提交
253
  dndCleanupVnodes(pDnode);
S
Shengliang Guan 已提交
254
  dndCleanupMgmt(pDnode);
S
Shengliang Guan 已提交
255
  tfsClose(pDnode->pTfs);
256

S
Shengliang Guan 已提交
257
  dndCloseImp(pDnode);
S
Shengliang Guan 已提交
258
  free(pDnode);
S
Shengliang Guan 已提交
259
  dInfo("dnode object is closed, data:%p", pDnode);
S
Shengliang Guan 已提交
260
}
S
Shengliang Guan 已提交
261 262 263 264 265 266 267 268 269 270 271 272

int32_t dndInit(const SDnodeEnvCfg *pCfg) {
  if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
    terrno = TSDB_CODE_REPEAT_INIT;
    dError("failed to init dnode env since %s", terrstr());
    return -1;
  }

  taosIgnSIGPIPE();
  taosBlockSIGPIPE();
  taosResolveCRC();

S
Shengliang Guan 已提交
273 274 275 276 277 278 279 280 281 282 283 284
  if (rpcInit() != 0) {
    dError("failed to init rpc since %s", terrstr());
    dndCleanup();
    return -1;
  }

  if (walInit() != 0) {
    dError("failed to init wal since %s", terrstr());
    dndCleanup();
    return -1;
  }

S
Shengliang Guan 已提交
285 286 287 288 289 290 291 292 293 294 295 296
  SVnodeOpt vnodeOpt = {
      .sver = pCfg->sver,
      .timezone = pCfg->timezone,
      .locale = pCfg->locale,
      .charset = pCfg->charset,
      .nthreads = pCfg->numOfCommitThreads,
      .putReqToVQueryQFp = dndPutReqToVQueryQ,
  };

  if (vnodeInit(&vnodeOpt) != 0) {
    dError("failed to init vnode since %s", terrstr());
    dndCleanup();
H
Hongze Cheng 已提交
297
    return -1;
S
Shengliang Guan 已提交
298 299
  }

S
Shengliang Guan 已提交
300 301 302 303 304 305 306 307 308 309 310
  memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg));
  dInfo("dnode env is initialized");
  return 0;
}

void dndCleanup() {
  if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
    dError("dnode env is already cleaned up");
    return;
  }

S
Shengliang Guan 已提交
311
  walCleanUp();
S
Shengliang Guan 已提交
312
  vnodeCleanup();
S
Shengliang Guan 已提交
313 314
  rpcCleanup();

S
Shengliang Guan 已提交
315 316
  taosStopCacheRefreshWorker();
  dInfo("dnode env is cleaned up");
S
Shengliang Guan 已提交
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
}

// OTHER FUNCTIONS ===================================
void taosGetDisk() {
#if 0  
  const double unit = 1024 * 1024 * 1024;
  
  SDiskSize    diskSize = tfsGetSize(pTfs);
  
  tfsUpdateSize(&fsMeta);
  tsTotalDataDirGB = (float)(fsMeta.total / unit);
  tsUsedDataDirGB = (float)(fsMeta.used / unit);
  tsAvailDataDirGB = (float)(fsMeta.avail / unit);

  if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
    tsTotalLogDirGB = (float)(diskSize.total / unit);
    tsAvailLogDirGB = (float)(diskSize.avail / unit);
  }

  if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
    tsTotalTmpDirGB = (float)(diskSize.total / unit);
    tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
  }
#endif
S
Shengliang Guan 已提交
341
}