refactor

src/common/inc/tdisk.h

@@ -26,81 +26,10 @@
 extern "C" {
 #endif
 
-typedef struct {
-  int level;
-  int did;
-} SDiskID;
-
-typedef struct {
-  uint64_t size;
-  uint64_t free;
-  uint64_t nfiles;
-} SDiskMeta;
-
-typedef struct {
-  uint64_t tsize;
-  uint64_t avail; // bytes
-} STiersMeta;
-
-typedef struct {
-  int       level;
-  int       did;
-  char      dir[TSDB_FILENAME_LEN];
-  SDiskMeta dmeta;
-} SDisk;
-
-typedef struct {
-  int    level;
-  int    nDisks;
-  SDisk *disks[TSDB_MAX_DISKS_PER_TIER];
-} STier;
-
-typedef struct SDnodeTier {
-  pthread_mutex_t lock;
-  STiersMeta      meta;
-  int             nTiers;
-  STier           tiers[TSDB_MAX_TIERS];
-  SHashObj *      map;
-} SDnodeTier;
-
-extern struct SDnodeTier *tsDnodeTier;
-#define DNODE_PRIMARY_DISK(pDnodeTier) (pDnodeTier)->tiers[0].disks[0]
-
-static FORCE_INLINE int tdLockTiers(SDnodeTier *pDnodeTier) {
-  int code = pthread_mutex_lock(&(pDnodeTier->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE int tdUnLockTiers(SDnodeTier *pDnodeTier) {
-  int code = pthread_mutex_unlock(&(pDnodeTier->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE SDisk *tdGetDisk(SDnodeTier *pDnodeTier, int level, int did) {
-  if (level < 0 || level >= pDnodeTier->nTiers) return NULL;
-
-  if (did < 0 || did >= pDnodeTier->tiers[level].nDisks) return NULL;
-
-  return pDnodeTier->tiers[level].disks[did];
-}
-
-SDnodeTier *tdNewTier();
-void *      tdCloseTier(SDnodeTier *pDnodeTier);
-int         tdAddDisks(SDnodeTier *pDnodeTier, SDiskCfg *pDiskCfgs, int ndisks);
-int         tdUpdateTiersInfo(SDnodeTier *pDnodeTier);
-int         tdCheckTiers(SDnodeTier *pDnodeTier);
-SDisk *     tdAssignDisk(SDnodeTier *pDnodeTier, int level);
-SDisk *     tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName);
-void        tdIncDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock);
-void        tdDecDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock);
+int  tdInitTiers(SDiskCfg *pDiskCfg, int ndisk);
+void tdDestroyTiers();
+int  tdUpdateDiskInfos();
+void tdGetPrimaryPath(char *dst);
 
 #ifdef __cplusplus
 }

src/common/src/tdisk.c

@@ -17,111 +17,139 @@
 #include "tdisk.h"
 #include "tulog.h"
 
+typedef struct {
+  int level;
+  int did;
+} SDiskID;
+
+typedef struct {
+  uint64_t size;
+  uint64_t free;
+  uint64_t nfiles;
+} SDiskMeta;
+
+typedef struct {
+  uint64_t tsize;
+  uint64_t avail; // bytes
+} STiersMeta;
+
+typedef struct {
+  int       level;
+  int       did;
+  char      dir[TSDB_FILENAME_LEN];
+  SDiskMeta dmeta;
+} SDisk;
+
+typedef struct {
+  int    level;
+  int    nDisks;
+  SDisk *disks[TSDB_MAX_DISKS_PER_TIER];
+} STier;
+
+typedef struct STiers {
+  pthread_mutex_t lock;
+  STiersMeta      meta;
+  int             nLevel;
+  STier           tiers[TSDB_MAX_TIERS];
+  SHashObj *      map;
+} STiers;
+
 #define DISK_MIN_FREE_SPACE 30 * 1024 * 1024  // disk free space less than 100M will not create new file again
 #define DNODE_DISK_AVAIL(pDisk) ((pDisk)->dmeta.free > DISK_MIN_FREE_SPACE)
+#define TIER_AT_LEVEL(level) (pTiers->tiers + (level))
+#define DISK_AT(level, did) (TIER_AT_LEVEL(level)->disks[(did)])
 
-static int tdFormatDir(char *idir, char *odir);
-static int tdCheckDisk(char *dirName, int level, int primary);
-static int tdUpdateDiskMeta(SDisk *pDisk);
-static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary);
+static struct STiers  tdTiers;
+static struct STiers *pTiers = &tdTiers;
 
-struct SDnodeTier *tsDnodeTier = NULL;
+int tdInitTiers(SDiskCfg *pDiskCfg, int ndisk) {
+  ASSERT(ndisk > 0);
 
-SDnodeTier *tdNewTier() {
-  SDnodeTier *pDnodeTier = (SDnodeTier *)calloc(1, sizeof(*pDnodeTier));
-  if (pDnodeTier == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return NULL;
-  }
+  memset((void *)pTiers, 0, sizeof(*pTiers));
 
-  int ret = pthread_mutex_init(&(pDnodeTier->lock), NULL);
+  int ret = pthread_mutex_init(&(pTiers->lock), NULL);
   if (ret != 0) {
     terrno = TAOS_SYSTEM_ERROR(ret);
-    tdCloseTier(pDnodeTier);
-    return NULL;
+    return -1;
   }
 
-  pDnodeTier->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
-                                 taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-  if (pDnodeTier->map == NULL) {
+  pTiers->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
+                             taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  if (pTiers->map == NULL) {
     terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    tdCloseTier(pDnodeTier);
-    return NULL;
+    tdDestroyTiers();
+    return -1;
   }
 
-  return pDnodeTier;
-}
-
-void *tdCloseTier(SDnodeTier *pDnodeTier) {
-  if (pDnodeTier) {
-    if (pDnodeTier->map) {
-      taosHashCleanup(pDnodeTier->map);
-      pDnodeTier->map = NULL;
+  for (int idisk = 0; idisk < ndisk; idisk++) {
+    if (tdAddDisk(pDiskCfg + idisk) < 0) {
+      tdDestroyTiers();
+      return -1;
     }
+  }
 
-    pthread_mutex_destroy(&(pDnodeTier->lock));
-
-    for (int i = 0; i < pDnodeTier->nTiers; i++) {
-      STier *pTier = pDnodeTier->tiers + i;
-      for (int j = 0; j < pTier->nDisks; j++) {
-        if (pTier->disks[j]) {
-          free(pTier->disks[j]);
-          pTier->disks[j] = NULL;
-        }
-      }
-    }
-    free(pDnodeTier);
+  if (tdCheckTiers() < 0) {
+    tdDestroyTiers();
+    return -1;
   }
-  return NULL;
+
+  return 0;
 }
 
-int tdAddDisks(SDnodeTier *pDnodeTier, SDiskCfg *pDiskCfgs, int ndisks) {
-  ASSERT(ndisks > 0);
+void tdDestroyTiers() {
+  taosHashCleanup(pTiers->map);
+  pTiers->map = NULL;
 
-  for (int i = 0; i < ndisks; i++) {
-    SDiskCfg *pCfg = pDiskCfgs + i;
-    tdAddDisk(pDnodeTier, pCfg->dir, pCfg->level, pCfg->primary);
-  }
+  pthread_mutex_destroy(&(pTiers->lock));
 
-  if (tdCheckTiers(pDnodeTier) < 0) return -1;
-  
-  return 0;
+  for (int level = TSDB_MAX_TIERS - 1; level >= 0; --level) {
+    for (int did = TSDB_MAX_DISKS_PER_TIER - 1; did >= 0; --did) {
+      if (DISK_AT(level, did)) {
+        free(DISK_AT(level, did));
+        DISK_AT(level, did) = NULL;
+      }
+    }
+  }
 }
 
-int tdUpdateTiersInfo(SDnodeTier *pDnodeTier) {
-  tdLockTiers(pDnodeTier);
+int tdUpdateDiskInfos() {
+  tdLockTiers(pTiers);
 
-  pDnodeTier->meta.tsize = 0;
-  pDnodeTier->meta.avail = 0;
+  pTiers->meta.tsize = 0;
+  pTiers->meta.avail = 0;
 
-  for (int i = 0; i < pDnodeTier->nTiers; i++) {
-    STier *pTier = pDnodeTier->tiers + i;
+  for (int i = 0; i < pTiers->nLevel; i++) {
+    STier *pTier = pTiers->tiers + i;
 
     for (int j = 0; j < pTier->nDisks; j++) {
       SDisk *pDisk = pTier->disks[j];
       if (tdUpdateDiskMeta(pDisk) < 0) {
-        tdUnLockTiers(pDnodeTier);
+        tdUnLockTiers(pTiers);
         return -1;
       }
 
-      pDnodeTier->meta.tsize += pDisk->dmeta.size;
-      pDnodeTier->meta.avail += pDisk->dmeta.free;
+      pTiers->meta.tsize += pDisk->dmeta.size;
+      pTiers->meta.avail += pDisk->dmeta.free;
     }
   }
 
-  tdUnLockTiers(pDnodeTier);
+  tdUnLockTiers(pTiers);
   return 0;
 }
 
-int tdCheckTiers(SDnodeTier *pDnodeTier) {
-  ASSERT(pDnodeTier->nTiers > 0);
-  if (DNODE_PRIMARY_DISK(pDnodeTier) == NULL) {
+void tdGetPrimaryPath(char *dst) { strncpy(dst, DISK_AT(0, 0)->dir, TSDB_FILENAME_LEN); }
+
+static SDisk *tdGetPrimaryDisk() { return DISK_AT(0, 0); }
+
+static int tdCheckTiers() {
+  ASSERT(pTiers->nLevel > 0);
+  if (tdGetPrimaryDisk(pTiers) == NULL) {
     terrno = TSDB_CODE_DND_LACK_PRIMARY_DISK;
     return -1;
   }
 
-  for (int i = 0; i < pDnodeTier->nTiers; i++) {
-    if (pDnodeTier->tiers[i].nDisks == 0) {
+  for (int i = 0; i < pTiers->nLevel; i++) {
+    if (pTiers->tiers[i].nDisks == 0) {
       terrno = TSDB_CODE_DND_NO_DISK_AT_TIER;
       return -1;
     }
@@ -130,15 +158,15 @@ int tdCheckTiers(SDnodeTier *pDnodeTier) {
   return 0;
 }
 
-SDisk *tdAssignDisk(SDnodeTier *pDnodeTier, int level) {
-  ASSERT(level < pDnodeTier->nTiers);
+static SDisk *tdAssignDisk(int level) {
+  ASSERT(level < pTiers->nLevel);
 
-  STier *pTier = pDnodeTier->tiers + level;
+  STier *pTier = pTiers->tiers + level;
   SDisk *pDisk = NULL;
 
   ASSERT(pTier->nDisks > 0);
 
-  tdLockTiers(pDnodeTier);
+  tdLockTiers(pTiers);
 
   for (int i = 0; i < pTier->nDisks; i++) {
     SDisk *iDisk = pTier->disks[i];
@@ -152,18 +180,18 @@ SDisk *tdAssignDisk(SDnodeTier *pDnodeTier, int level) {
 
   if (pDisk == NULL) {
     terrno = TSDB_CODE_DND_NO_DISK_SPACE;
-    tdUnLockTiers(pDnodeTier);
+    tdUnLockTiers(pTiers);
     return NULL;
   }
 
-  tdIncDiskFiles(pDnodeTier, pDisk, false);
+  tdIncDiskFiles(pTiers, pDisk, false);
 
-  tdUnLockTiers(pDnodeTier);
+  tdUnLockTiers(pTiers);
 
   return pDisk;
 }
 
-SDisk *tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName) {
+static SDisk *tdGetDiskByName(char *dirName) {
   char     fdirName[TSDB_FILENAME_LEN] = "\0";
   SDiskID *pDiskID = NULL;
 
@@ -171,34 +199,34 @@ SDisk *tdGetDiskByName(SDnodeTier *pDnodeTier, char *dirName) {
     return NULL;
   }
 
-  void *ptr = taosHashGet(pDnodeTier->map, (void *)fdirName, strnlen(fdirName, TSDB_FILENAME_LEN));
+  void *ptr = taosHashGet(pTiers->map, (void *)fdirName, strnlen(fdirName, TSDB_FILENAME_LEN));
   if (ptr == NULL) return NULL;
   pDiskID = (SDiskID *)ptr;
 
-  return tdGetDisk(pDnodeTier, pDiskID->level, pDiskID->did);
+  return tdGetDisk(pTiers, pDiskID->level, pDiskID->did);
 }
 
-void tdIncDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock) {
+static void tdIncDiskFiles(SDisk *pDisk, bool lock) {
   if (lock) {
-    tdLockTiers(pDnodeTier);
+    tdLockTiers(pTiers);
   }
 
   pDisk->dmeta.nfiles++;
 
   if (lock) {
-    tdUnLockTiers(pDnodeTier);
+    tdUnLockTiers(pTiers);
   }
 }
 
-void tdDecDiskFiles(SDnodeTier *pDnodeTier, SDisk *pDisk, bool lock) {
+static void tdDecDiskFiles(SDisk *pDisk, bool lock) {
   if (lock) {
-    tdLockTiers(pDnodeTier);
+    tdLockTiers(pTiers);
   }
 
   pDisk->dmeta.nfiles--;
 
   if (lock) {
-    tdUnLockTiers(pDnodeTier);
+    tdUnLockTiers(pTiers);
   }
 }
 
@@ -223,7 +251,7 @@ static int tdFormatDir(char *idir, char *odir) {
   return 0;
 }
 
-static int tdCheckDisk(char *dirName, int level, int primary) {
+static int tdCheckDisk(char *dirName) {
   if (access(dirName, W_OK | R_OK | F_OK) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -257,66 +285,66 @@ static int tdUpdateDiskMeta(SDisk *pDisk) {
   return 0;
 }
 
-static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary) {
+static int tdAddDisk(SDiskCfg *pCfg) {
   char    dirName[TSDB_FILENAME_LEN] = "\0";
   STier * pTier = NULL;
   SDiskID diskid = {0};
   SDisk * pDisk = NULL;
 
-  if (level < 0 || level >= TSDB_MAX_TIERS) {
+  if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) {
     terrno = TSDB_CODE_DND_INVALID_DISK_TIER;
-    uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+    uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
     return -1;
   }
 
-  if (tdFormatDir(dir, dirName) < 0) {
-    uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+  if (tdFormatDir(pCfg->dir, dirName) < 0) {
+    uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
     return -1;
   }
 
-  pTier = pDnodeTier->tiers + level;
-  diskid.level = level;
+  pTier = TIER_AT_LEVEL(pCfg->level);
+  diskid.level = pCfg->level;
 
   if (pTier->nDisks >= TSDB_MAX_DISKS_PER_TIER) {
     terrno = TSDB_CODE_DND_TOO_MANY_DISKS;
-    uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+    uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
     return -1;
   }
 
-  if (tdGetDiskByName(pDnodeTier, dirName) != NULL) {
+  if (tdGetDiskByName(dirName) != NULL) {
     terrno = TSDB_CODE_DND_DISK_ALREADY_EXISTS;
-    uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+    uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
     return -1;
   }
 
-  if (tdCheckDisk(dirName, level, primary) < 0) {
-    uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+  if (tdCheckDisk(dirName) < 0) {
+    uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
     return -1;
   }
 
-  if (primary) {
-    if (level != 0) {
+  if (pCfg->primary) {
+    if (pCfg->level != 0) {
       terrno = TSDB_CODE_DND_INVALID_DISK_TIER;
-      uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+      uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
       return -1;
     }
 
-    if (DNODE_PRIMARY_DISK(pDnodeTier) != NULL) {
+    if (tdGetPrimaryDisk() != NULL) {
       terrno = TSDB_CODE_DND_DUPLICATE_PRIMARY_DISK;
-      uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+      uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
       return -1;
     }
 
     diskid.did = 0;
   } else {
     if (level == 0) {
-      if (DNODE_PRIMARY_DISK(pDnodeTier) != NULL) {
+      if (tdGetPrimaryDisk() != NULL) {
         diskid.did = pTier->nDisks;
       } else {
         diskid.did = pTier->nDisks + 1;
         if (diskid.did >= TSDB_MAX_DISKS_PER_TIER) {
           terrno = TSDB_CODE_DND_TOO_MANY_DISKS;
-          uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+          uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
           return -1;
         }
       }
@@ -328,7 +356,7 @@ static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary)
   pDisk = (SDisk *)calloc(1, sizeof(SDisk));
   if (pDisk == NULL) {
     terrno = TSDB_CODE_DND_OUT_OF_MEMORY;
-    uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+    uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
     return -1;
   }
 
@@ -336,27 +364,27 @@ static int tdAddDisk(SDnodeTier *pDnodeTier, char *dir, int level, int primary)
   pDisk->level = diskid.level;
   pDisk->did = diskid.did;
 
-  if (taosHashPut(pDnodeTier->map, (void *)dirName, strnlen(dirName, TSDB_FILENAME_LEN), (void *)(&diskid),
+  if (taosHashPut(pTiers->map, (void *)dirName, strnlen(dirName, TSDB_FILENAME_LEN), (void *)(&diskid),
                   sizeof(diskid)) < 0) {
     free(pDisk);
     terrno = TSDB_CODE_DND_OUT_OF_MEMORY;
-    uError("failed to add disk %s to tier %d level since %s", dir, level, tstrerror(terrno));
+    uError("failed to add disk %s to tier %d level since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
     return -1;
   }
 
   pTier->nDisks++;
   pTier->disks[diskid.did] = pDisk;
-  pDnodeTier->nTiers = MAX(pDnodeTier->nTiers, level + 1);
+  pTiers->nLevel = MAX(pTiers->nLevel, level + 1);
 
   return 0;
 }
 
-void taosGetDisk() {
+static void taosGetDisk() {
   const double unit = 1024 * 1024 * 1024;
   SysDiskSize  diskSize;
 
   if (tscEmbedded) {
-    tdUpdateTiersInfo(tsDnodeTier);
+    tdUpdateDiskInfos(tsDnodeTier);
     tsTotalDataDirGB = (float)tsDnodeTier->meta.tsize / unit;
     tsAvailDataDirGB = (float)tsDnodeTier->meta.avail / unit;
   }
@@ -371,3 +399,29 @@ void taosGetDisk() {
     tsAvailTmpDirectorySpace = (float)diskSize.avail / unit;
   }
 }
+
+static int tdLockTiers() {
+  int code = pthread_mutex_lock(&(pTiers->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static int tdUnLockTiers() {
+  int code = pthread_mutex_unlock(&(pTiers->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static SDisk *tdGetDisk(int level, int did) {
+  if (level < 0 || level >= pTiers->nLevel) return NULL;
+
+  if (did < 0 || did >= pTiers->tiers[level].nDisks) return NULL;
+
+  return pTiers->tiers[level].disks[did];
+}
\ No newline at end of file

src/dnode/src/dnodeMain.c

@@ -183,17 +183,11 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
-  tsDnodeTier = tdNewTier();
-  if (tsDnodeTier == NULL) {
-    dError("failed to create new dnode tier since %s", tstrerror(terrno));
-    return -1;
-  }
-
-  if (tdAddDisks(tsDnodeTier, tsDiskCfg, tsDiskCfgNum) < 0) {
+  if (tdInitTiers(tsDiskCfg, tsDiskCfgNum) < 0) {
     dError("failed to add disks to dnode tier since %s", tstrerror(terrno));
     return -1;
   }
-  strncpy(tsDataDir, DNODE_PRIMARY_DISK(tsDnodeTier)->dir, TSDB_FILENAME_LEN);
+  tdGetPrimaryPath(tsDataDir);
   tdGetVnodeRootDir(tsDataDir, tsVnodeDir);
 
   //TODO(dengyihao): no need to init here 
@@ -236,12 +230,7 @@ static int32_t dnodeInitStorage() {
   return 0;
 }
 
-static void dnodeCleanupStorage() {
-  if (tsDnodeTier) {
-    tdCloseTier(tsDnodeTier);
-    tsDnodeTier = NULL;
-  }
-}
+static void dnodeCleanupStorage() { tdDestroyTiers(); }
 
 bool  dnodeIsFirstDeploy() {
   return strcmp(tsFirst, tsLocalEp) == 0;