From 44963201abead1cef70f62c89faeeadf99bbe18e Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 22 Oct 2020 06:28:44 +0000
Subject: [PATCH] TD-1767

---
 CMakeLists.txt                |  1 +
 cmake/define.inc              |  4 ++++
 src/common/inc/tglobal.h      | 18 +++++++++++------
 src/common/src/tglobal.c      | 38 ++++++++++++++++++++++++++++++++++-
 src/dnode/src/dnodeMain.c     | 20 +++++++++---------
 src/dnode/src/dnodeTier.c     |  8 ++++----
 src/inc/dnode.h               | 18 +++++------------
 src/inc/taosdef.h             |  4 ++++
 src/os/src/detail/osSysinfo.c |  1 -
 src/tsdb/src/tsdbFile.c       |  6 +++---
 src/util/CMakeLists.txt       |  4 ++++
 src/util/inc/tconfig.h        |  1 +
 src/util/src/tconfig.c        | 27 ++++++++++++++++---------
 src/vnode/src/vnodeMain.c     |  4 ++--
 14 files changed, 104 insertions(+), 50 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 588526c286..d52f31124a 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -16,6 +16,7 @@ SET(TD_GRANT FALSE)
 SET(TD_SYNC TRUE)
 SET(TD_MQTT TRUE)
 SET(TD_TSDB_PLUGINS FALSE)
+SET(TD_STORAGE FALSE)
 
 SET(TD_COVER FALSE)
 SET(TD_MEM_CHECK FALSE)
diff --git a/cmake/define.inc b/cmake/define.inc
index 6e64c2709a..f3b579e789 100755
--- a/cmake/define.inc
+++ b/cmake/define.inc
@@ -25,6 +25,10 @@ IF (TD_TSDB_PLUGINS)
   ADD_DEFINITIONS(-D_TSDB_PLUGINS)
 ENDIF ()
 
+IF (TD_STORAGE)
+  ADD_DEFINITIONS(-D_STORAGE)
+ENDIF ()
+
 IF (TD_GODLL)
   ADD_DEFINITIONS(-D_TD_GO_DLL_)
 ENDIF ()
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index 054c894251..401e64acec 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -20,8 +20,6 @@
 extern "C" {
 #endif
 
-struct SDnodeTier;
-
 // cluster
 extern char     tsFirst[];
 extern char     tsSecond[];
@@ -157,9 +155,6 @@ extern char gitinfo[];
 extern char gitinfoOfInternal[];
 extern char buildinfo[];
 
-// dnode
-extern struct SDnodeTier *pDnodeTier;
-
 // log
 extern int32_t tsAsyncLog;
 extern int32_t tsNumOfLogLines;
@@ -182,6 +177,14 @@ extern int32_t wDebugFlag;
 extern int32_t cqDebugFlag;
 extern int32_t debugFlag;
 
+typedef struct {
+  char dir[TSDB_FILENAME_LEN];
+  int  level;
+  int  primary;
+} SDiskCfg;
+extern int32_t  tsDiskCfgNum;
+extern SDiskCfg tsDiskCfg[];
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 void taosInitGlobalCfg();
@@ -190,7 +193,10 @@ void taosSetAllDebugFlag();
 bool taosCfgDynamicOptions(char *msg);
 int  taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
 bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
- 
+void taosAddDataDir(int index, char *v1, int level, int primary);
+void taosReadDataDirCfg(char *v1, char *v2, char *v3);
+void taosPrintDataDirCfg();
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index c02c8af1e4..bf824f0dbd 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -160,6 +160,13 @@ char    tsDnodeDir[TSDB_FILENAME_LEN] = {0};
 char    tsMnodeDir[TSDB_FILENAME_LEN] = {0};
 char    tsDataDir[TSDB_FILENAME_LEN] = {0};
 char    tsScriptDir[TSDB_FILENAME_LEN] = {0};
+int32_t tsDiskCfgNum = 0;
+
+#ifndef _STORAGE
+SDiskCfg tsDiskCfg[1];
+#else
+SDiskCfg tsDiskCfg[TSDB_MAX_DISKS];
+#endif
 
 /*
  * minimum scale for whole system, millisecond by default
@@ -305,6 +312,34 @@ bool taosCfgDynamicOptions(char *msg) {
   return false;
 }
 
+void taosAddDataDir(int index, char *v1, int level, int primary) {
+  tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
+  tsDiskCfg[index].level = level;
+  tsDiskCfg[index].primary = primary;
+  uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
+}
+
+#ifndef _STORAGE
+void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
+  taosAddDataDir(0, tsDataDir, 0, 1);
+  tstrncpy(tsDiskCfg[0].dir, tsDataDir, TSDB_FILENAME_LEN);
+}
+#endif
+
+void taosPrintDataDirCfg() {
+  for (int i = 0; i < tsDiskCfgNum; ++i) {
+    SDiskCfg *cfg = &tsDiskCfg[i];
+    uInfo(" dataDir:%s level:%d primary:%d", cfg->dir, cfg->level, cfg->primary);
+  }
+}
+
+static void taosCheckDataDirCfg() {
+  if (tsDiskCfgNum <= 0) {
+    taosAddDataDir(0, tsDataDir, 0, 1);
+    tsDiskCfgNum = 1;
+  }
+}
+
 static void doInitGlobalConfig(void) {
   osInit();
   srand(taosSafeRand());
@@ -386,7 +421,7 @@ static void doInitGlobalConfig(void) {
 
   cfg.option = "dataDir";
   cfg.ptr = tsDataDir;
-  cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
+  cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -1328,6 +1363,7 @@ bool taosCheckGlobalCfg() {
     snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port);
   }
 
+  taosCheckDataDirCfg();
   taosGetSystemInfo();
 
   tsSetLocale();
diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c
index 402c4ebbd2..9beaabe888 100644
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@@ -169,17 +169,17 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
-  pDnodeTier = dnodeNewTier();
-  if (pDnodeTier == NULL) {
+  tsDnodeTier = dnodeNewTier();
+  if (tsDnodeTier == NULL) {
     dError("failed to create new dnode tier since %s", tstrerror(terrno));
     return -1;
   }
 
-  if (dnodeAddDisks(pDnodeTier, NULL, 0) < 0) {
+  if (dnodeAddDisks(tsDnodeTier, tsDiskCfg, tsDiskCfgNum) < 0) {
     dError("failed to add disks to dnode tier since %s", tstrerror(terrno));
     return -1;
   }
-  strncpy(tsDataDir, DNODE_PRIMARY_DISK(pDnodeTier)->dir, TSDB_FILENAME_LEN);
+  strncpy(tsDataDir, DNODE_PRIMARY_DISK(tsDnodeTier)->dir, TSDB_FILENAME_LEN);
   tdGetVnodeRootDir(tsDataDir, tsVnodeDir);
 
   //TODO(dengyihao): no need to init here 
@@ -195,12 +195,12 @@ static int32_t dnodeInitStorage() {
    return -1;
   }
 
-  for (int i = 0; i < pDnodeTier->nTiers; i++) {
+  for (int i = 0; i < tsDnodeTier->nTiers; i++) {
     char dirName[TSDB_FILENAME_LEN];
 
-    STier *pTier = pDnodeTier->tiers + i;
+    STier *pTier = tsDnodeTier->tiers + i;
     for (int j = 0; j < pTier->nDisks; j++) {
-      SDisk *pDisk = dnodeGetDisk(pDnodeTier, i, j);
+      SDisk *pDisk = dnodeGetDisk(tsDnodeTier, i, j);
 
       tdGetVnodeRootDir(dirName, pDisk->dir);
       if (dnodeCreateDir(dirName) < 0) {
@@ -223,9 +223,9 @@ static int32_t dnodeInitStorage() {
 }
 
 static void dnodeCleanupStorage() {
-  if (pDnodeTier) {
-    dnodeCloseTier(pDnodeTier);
-    pDnodeTier = NULL;
+  if (tsDnodeTier) {
+    dnodeCloseTier(tsDnodeTier);
+    tsDnodeTier = NULL;
   }
 }
 
diff --git a/src/dnode/src/dnodeTier.c b/src/dnode/src/dnodeTier.c
index 942fd9bf82..c0b1a16e5f 100644
--- a/src/dnode/src/dnodeTier.c
+++ b/src/dnode/src/dnodeTier.c
@@ -40,7 +40,7 @@ SDnodeTier *dnodeNewTier() {
     return NULL;
   }
 
-  pDnodeTier->map = taosHashInit(DNODE_MAX_TIERS * DNODE_MAX_DISKS_PER_TIER * 2,
+  pDnodeTier->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
                                  taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
   if (pDnodeTier->map == NULL) {
     terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
@@ -217,7 +217,7 @@ static int dnodeAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primar
   SDiskID diskid = {0};
   SDisk * pDisk = NULL;
 
-  if (level < 0 || level >= DNODE_MAX_TIERS) {
+  if (level < 0 || level >= TSDB_MAX_TIERS) {
     terrno = TSDB_CODE_DND_INVALID_DISK_TIER;
     dError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
     return -1;
@@ -231,7 +231,7 @@ static int dnodeAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primar
   pTier = pDnodeTier->tiers + level;
   diskid.level = level;
 
-  if (pTier->nDisks >= DNODE_MAX_DISKS_PER_TIER) {
+  if (pTier->nDisks >= TSDB_MAX_DISKS_PER_TIER) {
     terrno = TSDB_CODE_DND_TOO_MANY_DISKS;
     dError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
     return -1;
@@ -268,7 +268,7 @@ static int dnodeAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primar
         diskid.did = pTier->nDisks;
       } else {
         diskid.did = pTier->nDisks + 1;
-        if (diskid.did >= DNODE_MAX_DISKS_PER_TIER) {
+        if (diskid.did >= TSDB_MAX_DISKS_PER_TIER) {
           terrno = TSDB_CODE_DND_TOO_MANY_DISKS;
           dError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
           return -1;
diff --git a/src/inc/dnode.h b/src/inc/dnode.h
index ca721542cc..8570f78fd1 100644
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@@ -20,6 +20,8 @@
 extern "C" {
 #endif
 
+#include "taosdef.h"
+#include "tglobal.h"
 #include "hash.h"
 #include "taoserror.h"
 #include "trpc.h"
@@ -71,16 +73,6 @@ void    dnodeDelayReprocessMnodeWriteMsg(void *pMsg);
 
 void    dnodeSendStatusMsgToMnode();
 
-// DNODE TIER
-#define DNODE_MAX_TIERS 3
-#define DNODE_MAX_DISKS_PER_TIER 16
-
-typedef struct {
-  char dir[TSDB_FILENAME_LEN];
-  int  level;
-  int  primary;
-} SDiskCfg;
-
 typedef struct {
   int level;
   int did;
@@ -100,16 +92,17 @@ typedef struct {
 typedef struct {
   int    level;
   int    nDisks;
-  SDisk *disks[DNODE_MAX_DISKS_PER_TIER];
+  SDisk *disks[TSDB_MAX_DISKS_PER_TIER];
 } STier;
 
 typedef struct SDnodeTier {
   pthread_rwlock_t rwlock;
   int              nTiers;
-  STier            tiers[DNODE_MAX_TIERS];
+  STier            tiers[TSDB_MAX_TIERS];
   SHashObj *       map;
 } SDnodeTier;
 
+extern struct SDnodeTier *tsDnodeTier;
 #define DNODE_PRIMARY_DISK(pDnodeTier) (pDnodeTier)->tiers[0].disks[0]
 
 static FORCE_INLINE int dnodeRLockTiers(SDnodeTier *pDnodeTier) {
@@ -155,7 +148,6 @@ int         dnodeCheckTiers(SDnodeTier *pDnodeTier);
 SDisk *     dnodeAssignDisk(SDnodeTier *pDnodeTier, int level);
 SDisk *     dnodeGetDiskByName(SDnodeTier *pDnodeTier, char *dirName);
 
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index aee60da201..162be1583f 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -432,6 +432,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TAOS_QTYPE_CQ       3
 #define TAOS_QTYPE_QUERY    4
 
+#define TSDB_MAX_TIERS           3
+#define TSDB_MAX_DISKS_PER_TIER 16
+#define TSDB_MAX_DISKS           (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER)
+
 typedef enum {
   TSDB_SUPER_TABLE        = 0,  // super table
   TSDB_CHILD_TABLE        = 1,  // table created from super table
diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c
index 8df671f9c8..befd6f1cbd 100644
--- a/src/os/src/detail/osSysinfo.c
+++ b/src/os/src/detail/osSysinfo.c
@@ -533,7 +533,6 @@ void taosPrintOsInfo() {
   uInfo(" os release:             %s", buf.release);
   uInfo(" os version:             %s", buf.version);
   uInfo(" os machine:             %s", buf.machine);
-  uInfo("==================================");
 }
 
 void taosKillSystem() {
diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index be3190a622..851991a70d 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -26,7 +26,7 @@
 #include "dnode.h"
 #include "tpath.h"
 
-struct SDnodeTier *pDnodeTier = NULL;
+struct SDnodeTier *tsDnodeTier = NULL;
 const char *       tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"};
 
 static void      tsdbDestroyFile(SFile *pFile);
@@ -79,8 +79,8 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
   ASSERT(pRepo != NULL && pRepo->tsdbFileH != NULL);
   char dataDir[TSDB_FILENAME_LEN] = "\0";
 
-  for (int level = 0; level < pDnodeTier->nTiers; level++) {
-    STier *pTier = pDnodeTier->tiers + level;
+  for (int level = 0; level < tsDnodeTier->nTiers; level++) {
+    STier *pTier = tsDnodeTier->tiers + level;
     for (int did = 0; did < pTier->nDisks; did++) {
       SDisk *pDisk = pTier->disks[did];
 
diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt
index 89c8e3dc39..df4ef0d78d 100644
--- a/src/util/CMakeLists.txt
+++ b/src/util/CMakeLists.txt
@@ -29,3 +29,7 @@ ELSEIF (TD_WINDOWS)
 ELSEIF(TD_DARWIN)
   TARGET_LINK_LIBRARIES(tutil iconv)
 ENDIF()
+
+IF (TD_STORAGE)
+  TARGET_LINK_LIBRARIES(tutil storage)
+ENDIF ()
\ No newline at end of file
diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h
index 0520cf29a8..8bd846531f 100644
--- a/src/util/inc/tconfig.h
+++ b/src/util/inc/tconfig.h
@@ -47,6 +47,7 @@ enum {
   TAOS_CFG_VTYPE_STRING,
   TAOS_CFG_VTYPE_IPSTR,
   TAOS_CFG_VTYPE_DIRECTORY,
+  TAOS_CFG_VTYPE_DATA_DIRCTORY,
 };
 
 enum {
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index 0ec55841a0..80d911f3fc 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -199,7 +199,7 @@ SGlobalCfg *taosGetConfigOption(const char *option) {
   return NULL;
 }
 
-static void taosReadConfigOption(const char *option, char *value) {
+static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) {
   for (int i = 0; i < tsGlobalConfigNum; ++i) {
     SGlobalCfg *cfg = tsGlobalConfig + i;
     if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue;
@@ -224,6 +224,9 @@ static void taosReadConfigOption(const char *option, char *value) {
       case TAOS_CFG_VTYPE_DIRECTORY:
         taosReadDirectoryConfig(cfg, value);
         break;
+      case TAOS_CFG_VTYPE_DATA_DIRCTORY:
+        taosReadDirectoryConfig(cfg, value);
+        taosReadDataDirCfg(value, value2, value3);
       default:
         uError("config option:%s, input value:%s, can't be recognized", option, value);
         break;
@@ -307,8 +310,8 @@ void taosReadGlobalLogCfg() {
 }
 
 bool taosReadGlobalCfg() {
-  char * line, *option, *value, *value1;
-  int    olen, vlen, vlen1;
+  char * line, *option, *value, *value2, *value3;
+  int    olen, vlen, vlen2, vlen3;
   char   fileName[PATH_MAX] = {0};
 
   sprintf(fileName, "%s/taos.cfg", configDir);
@@ -331,8 +334,8 @@ bool taosReadGlobalCfg() {
   while (!feof(fp)) {
     memset(line, 0, len);
 
-    option = value = NULL;
-    olen = vlen = 0;
+    option = value = value2 = value3 = NULL;
+    olen = vlen = vlen2 = vlen3 = 0;
 
     taosGetline(&line, &len, fp);
     line[len - 1] = 0;
@@ -345,11 +348,13 @@ bool taosReadGlobalCfg() {
     if (vlen == 0) continue;
     value[vlen] = 0;
 
-    // For dataDir, the format is:
-    // dataDir    /mnt/disk1    0
-    paGetToken(value + vlen + 1, &value1, &vlen1);
-    
-    taosReadConfigOption(option, value);
+    paGetToken(value + vlen + 1, &value2, &vlen2);
+    if (vlen2 != 0) value2[vlen2] = 0;
+
+    paGetToken(value + vlen2 + 1, &value3, &vlen3);
+    if (vlen3 != 0) value3[vlen3] = 0;
+
+    taosReadConfigOption(option, value, value2, value3);
   }
 
   fclose(fp);
@@ -397,4 +402,6 @@ void taosPrintGlobalCfg() {
   }
 
   taosPrintOsInfo();
+  taosPrintDataDirCfg();
+  uInfo("==================================");
 }
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 989cd2adcb..747c04c37d 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -418,8 +418,8 @@ void vnodeRelease(void *pVnodeRaw) {
     char rootDir[TSDB_FILENAME_LEN] = {0};    
     char newDir[TSDB_FILENAME_LEN] = {0};
 
-    for (int i = 0; i < pDnodeTier->nTiers; i++) {
-      STier *pTier = pDnodeTier->tiers + i;
+    for (int i = 0; i < tsDnodeTier->nTiers; i++) {
+      STier *pTier = tsDnodeTier->tiers + i;
       for (int j = 0; j < pTier->nDisks; j++) {
         SDisk *pDisk = pTier->disks[j];
 
-- 
GitLab