Merge pull request #9886 from taosdata/feature/tfs

Feature/tfs

include/libs/tfs/tfs.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TD_TFS_H
-#define TD_TFS_H
+#ifndef _TD_TFS_H_
+#define _TD_TFS_H_
 
 #include "tglobal.h"
 
@@ -23,8 +23,8 @@ extern "C" {
 #endif
 
 typedef struct {
-  int level;
-  int id;
+  int32_t level;
+  int32_t id;
 } SDiskID;
 
 #define TFS_UNDECIDED_LEVEL -1
@@ -36,31 +36,23 @@ typedef struct {
 
 // FS APIs ====================================
 typedef struct {
-  int64_t tsize;
+  int64_t total;
   int64_t used;
   int64_t avail;
 } SFSMeta;
 
-typedef struct {
-  int64_t size;
-  int64_t used;
-  int64_t free;
-  int16_t nAvailDisks;  // # of Available disks
-} STierMeta;
-
-int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
+int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk);
 void    tfsCleanup();
-void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels);
-void tfsGetMeta(SFSMeta *pMeta);
-void tfsAllocDisk(int expLevel, int *level, int *id);
+void    tfsUpdateSize(SFSMeta *pFSMeta);
+void    tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id);
 
 const char *TFS_PRIMARY_PATH();
-const char *TFS_DISK_PATH(int level, int id);
+const char *TFS_DISK_PATH(int32_t level, int32_t id);
 
 // TFILE APIs ====================================
 typedef struct {
-  int  level;
-  int  id;
+  int32_t level;
+  int32_t id;
   char    rname[TSDB_FILENAME_LEN];  // REL name
   char    aname[TSDB_FILENAME_LEN];  // ABS name
 } TFILE;
@@ -76,23 +68,23 @@ typedef struct {
 #define tfscopy(sf, df) taosCopyFile(TFILE_NAME(sf), TFILE_NAME(df))
 #define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df))
 
-void  tfsInitFile(TFILE *pf, int level, int id, const char *bname);
+void    tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname);
 bool    tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
-int   tfsEncodeFile(void **buf, TFILE *pf);
+int32_t tfsEncodeFile(void **buf, TFILE *pf);
 void   *tfsDecodeFile(void *buf, TFILE *pf);
 void    tfsbasename(const TFILE *pf, char *dest);
 void    tfsdirname(const TFILE *pf, char *dest);
 
 // DIR APIs ====================================
-int tfsMkdirAt(const char *rname, int level, int id);
-int tfsMkdirRecurAt(const char *rname, int level, int id);
-int tfsMkdir(const char *rname);
-int tfsRmdir(const char *rname);
-int tfsRename(char *orname, char *nrname);
+int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id);
+int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id);
+int32_t tfsMkdir(const char *rname);
+int32_t tfsRmdir(const char *rname);
+int32_t tfsRename(char *orname, char *nrname);
 
 typedef struct TDIR TDIR;
 
-TDIR *       tfsOpendir(const char *rname);
+TDIR        *tfsOpendir(const char *rname);
 const TFILE *tfsReaddir(TDIR *tdir);
 void         tfsClosedir(TDIR *tdir);
 
@@ -100,4 +92,4 @@ void         tfsClosedir(TDIR *tdir);
 }
 #endif
 
-#endif
+#endif /*_TD_TFS_H_*/

include/os/osSysinfo.h

@@ -35,12 +35,12 @@ extern char     tsLocale[];
 extern char     tsCharset[];            // default encode string
 
 typedef struct {
-  int64_t tsize;
+  int64_t total;
   int64_t used;
   int64_t avail;
-} SysDiskSize;
+} SDiskSize;
 
-int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize);
+int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
 int32_t taosGetCpuCores();
 void    taosGetSystemInfo();
 bool    taosReadProcIO(int64_t *rchars, int64_t *wchars);

include/util/taoserror.h

@@ -411,7 +411,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_WAL_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x1004)  //"WAL out of memory")
 
 // tfs
-#define TSDB_CODE_FS_OUT_OF_MEMORY              TAOS_DEF_ERROR_CODE(0, 0x2200)  //"tfs out of memory")
+#define TSDB_CODE_FS_APP_ERROR                  TAOS_DEF_ERROR_CODE(0, 0x2200)  //"tfs out of memory")
 #define TSDB_CODE_FS_INVLD_CFG                  TAOS_DEF_ERROR_CODE(0, 0x2201)  //"tfs invalid mount config")
 #define TSDB_CODE_FS_TOO_MANY_MOUNT             TAOS_DEF_ERROR_CODE(0, 0x2202)  //"tfs too many mount")
 #define TSDB_CODE_FS_DUP_PRIMARY                TAOS_DEF_ERROR_CODE(0, 0x2203)  //"tfs duplicate primary mount")

source/libs/tfs/inc/tfsint.h → source/libs/tfs/inc/tfsInt.h

@@ -13,19 +13,20 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TD_TFSINT_H
-#define TD_TFSINT_H
+#ifndef _TD_TFS_INT_H_
+#define _TD_TFS_INT_H_
 
-#include "tlog.h"
-#include "tglobal.h"
-#include "tfs.h"
-#include "tcoding.h"
+#include "os.h"
 
-#ifdef __cplusplus
-extern "C" {
-#endif
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tcoding.h"
+#include "tfs.h"
+#include "tglobal.h"
+#include "thash.h"
+#include "tlog.h"
 
-extern int fsDebugFlag;
+extern int32_t fsDebugFlag;
 
 // For debug purpose
 #define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }}
@@ -38,60 +39,44 @@ extern int fsDebugFlag;
 // Global Definitions
 #define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
 
-// tdisk.c ======================================================
-typedef struct {
-  int64_t size;
-  int64_t used;
-  int64_t free;
-} SDiskMeta;
-
 typedef struct SDisk {
-  int       level;
-  int       id;
-  char      dir[TSDB_FILENAME_LEN];
-  SDiskMeta dmeta;
+  int32_t   level;
+  int32_t   id;
+  char     *path;
+  SDiskSize size;
 } SDisk;
 
-#define DISK_LEVEL(pd) ((pd)->level)
-#define DISK_ID(pd) ((pd)->id)
-#define DISK_DIR(pd) ((pd)->dir)
-#define DISK_META(pd) ((pd)->dmeta)
-#define DISK_SIZE(pd) ((pd)->dmeta.size)
-#define DISK_USED_SIZE(pd) ((pd)->dmeta.used)
-#define DISK_FREE_SIZE(pd) ((pd)->dmeta.free)
-
-SDisk *tfsNewDisk(int level, int id, const char *dir);
-SDisk *tfsFreeDisk(SDisk *pDisk);
-int    tfsUpdateDiskInfo(SDisk *pDisk);
-
-// ttier.c ======================================================
-
 typedef struct STier {
   pthread_spinlock_t lock;
-  int                level;
-  int16_t            ndisk;   // # of disks mounted to this tier
+  int32_t            level;
   int16_t            nextid;       // next disk id to allocate
-  STierMeta          tmeta;
-  SDisk *            disks[TSDB_MAX_DISKS_PER_TIER];
+  int16_t            ndisk;        // # of disks mounted to this tier
+  int16_t            nAvailDisks;  // # of Available disks
+  SDisk             *disks[TSDB_MAX_DISKS_PER_TIER];
+  SDiskSize          size;
 } STier;
 
 #define TIER_LEVEL(pt) ((pt)->level)
 #define TIER_NDISKS(pt) ((pt)->ndisk)
 #define TIER_SIZE(pt) ((pt)->tmeta.size)
 #define TIER_FREE_SIZE(pt) ((pt)->tmeta.free)
-#define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks)
+
 #define DISK_AT_TIER(pt, id) ((pt)->disks[id])
+#define DISK_DIR(pd) ((pd)->path)
 
-int    tfsInitTier(STier *pTier, int level);
+SDisk  *tfsNewDisk(int32_t level, int32_t id, const char *dir);
+SDisk  *tfsFreeDisk(SDisk *pDisk);
+int32_t tfsUpdateDiskSize(SDisk *pDisk);
+
+int32_t tfsInitTier(STier *pTier, int32_t level);
 void    tfsDestroyTier(STier *pTier);
 SDisk  *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg);
-void   tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta);
-int    tfsAllocDiskOnTier(STier *pTier);
-void   tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta);
+void    tfsUpdateTierSize(STier *pTier);
+int32_t tfsAllocDiskOnTier(STier *pTier);
 void    tfsPosNextId(STier *pTier);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_TFS_INT_H_*/

source/libs/tfs/src/tfs.c

@@ -13,22 +13,17 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
-
-#include "taosdef.h"
-#include "taoserror.h"
-#include "tfs.h"
-#include "tfsint.h"
-#include "thash.h"
+#define _DEFAULT_SOURCE
+#include "tfsInt.h"
 
 #define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)
 
 typedef struct {
   pthread_spinlock_t lock;
   SFSMeta            meta;
-  int                nlevel;
+  int32_t            nlevel;
   STier              tiers[TSDB_MAX_TIERS];
-  SHashObj *         map;  // name to did map
+  SHashObj          *map;  // name to did map
 } SFS;
 
 typedef struct {
@@ -52,21 +47,24 @@ static SFS  tfs = {0};
 static SFS *pfs = &tfs;
 
 // STATIC DECLARATION
-static int    tfsMount(SDiskCfg *pCfg);
-static int    tfsCheck();
-static int    tfsCheckAndFormatCfg(SDiskCfg *pCfg);
-static int    tfsFormatDir(char *idir, char *odir);
+static int32_t tfsMount(SDiskCfg *pCfg);
+static int32_t tfsCheck();
+static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg);
+static int32_t tfsFormatDir(char *idir, char *odir);
 static SDisk  *tfsGetDiskByID(SDiskID did);
 static SDisk  *tfsGetDiskByName(const char *dir);
-static int    tfsOpendirImpl(TDIR *tdir);
+static int32_t tfsOpendirImpl(TDIR *tdir);
 static void    tfsInitDiskIter(SDiskIter *pIter);
 static SDisk  *tfsNextDisk(SDiskIter *pIter);
 
 // FS APIs ====================================
-int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
-  ASSERT(ndisk > 0);
+int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk) {
+  if (ndisk < 0) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return -1;
+  }
 
-  for (int level = 0; level < TSDB_MAX_TIERS; level++) {
+  for (int32_t level = 0; level < TSDB_MAX_TIERS; level++) {
     if (tfsInitTier(TFS_TIER_AT(level), level) < 0) {
       while (true) {
         level--;
@@ -84,12 +82,12 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
   pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
                           taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
   if (pfs->map == NULL) {
-    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     tfsCleanup();
     return -1;
   }
 
-  for (int idisk = 0; idisk < ndisk; idisk++) {
+  for (int32_t idisk = 0; idisk < ndisk; idisk++) {
     if (tfsMount(pDiskCfg + idisk) < 0) {
       tfsCleanup();
       return -1;
@@ -101,8 +99,8 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
     return -1;
   }
 
-  tfsUpdateInfo(NULL, NULL, 0);
-  for (int level = 0; level < TFS_NLEVEL(); level++) {
+  tfsUpdateSize(NULL);
+  for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
     tfsPosNextId(TFS_TIER_AT(level));
   }
 
@@ -114,32 +112,27 @@ void tfsCleanup() {
   pfs->map = NULL;
 
   pthread_spin_destroy(&(pfs->lock));
-  for (int level = 0; level < TFS_NLEVEL(); level++) {
+  for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
     tfsDestroyTier(TFS_TIER_AT(level));
   }
 }
 
-void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) {
-  SFSMeta   fsMeta;
-  STierMeta tierMeta;
+void tfsUpdateSize(SFSMeta *pFSMeta) {
+  SFSMeta   fsMeta = {0};
+  SDiskSize size = {0};
 
   if (pFSMeta == NULL) {
     pFSMeta = &fsMeta;
   }
 
-  memset(pFSMeta, 0, sizeof(*pFSMeta));
-
-  for (int level = 0; level < TFS_NLEVEL(); level++) {
-    STierMeta *pTierMeta = &tierMeta;
-    if (tierMetas && level < numTiers) {
-      pTierMeta = tierMetas + level;
-    }
+  memset(pFSMeta, 0, sizeof(SFSMeta));
 
+  for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
     STier *pTier = TFS_TIER_AT(level);
-    tfsUpdateTierInfo(pTier, pTierMeta);
-    pFSMeta->tsize += pTierMeta->size;
-    pFSMeta->avail += pTierMeta->free;
-    pFSMeta->used += pTierMeta->used;
+    tfsUpdateTierSize(pTier);
+    pFSMeta->total += pTier->size.total;
+    pFSMeta->avail += pTier->size.avail;
+    pFSMeta->used += pTier->size.used;
   }
 
   tfsLock();
@@ -147,17 +140,9 @@ void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) {
   tfsUnLock();
 }
 
-void tfsGetMeta(SFSMeta *pMeta) {
-  ASSERT(pMeta);
-
-  tfsLock();
-  *pMeta = pfs->meta;
-  tfsUnLock();
-}
-
 /* Allocate an existing available tier level
  */
-void tfsAllocDisk(int expLevel, int *level, int *id) {
+void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id) {
   ASSERT(expLevel >= 0);
 
   *level = expLevel;
@@ -182,10 +167,10 @@ void tfsAllocDisk(int expLevel, int *level, int *id) {
 }
 
 const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); }
-const char *TFS_DISK_PATH(int level, int id) { return DISK_DIR(TFS_DISK_AT(level, id)); }
+const char *TFS_DISK_PATH(int32_t level, int32_t id) { return DISK_DIR(TFS_DISK_AT(level, id)); }
 
 // TFILE APIs ====================================
-void tfsInitFile(TFILE *pf, int level, int id, const char *bname) {
+void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname) {
   ASSERT(TFS_IS_VALID_DISK(level, id));
 
   SDisk *pDisk = TFS_DISK_AT(level, id);
@@ -208,8 +193,8 @@ bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) {
   return true;
 }
 
-int tfsEncodeFile(void **buf, TFILE *pf) {
-  int tlen = 0;
+int32_t tfsEncodeFile(void **buf, TFILE *pf) {
+  int32_t tlen = 0;
 
   tlen += taosEncodeVariantI32(buf, pf->level);
   tlen += taosEncodeVariantI32(buf, pf->id);
@@ -220,7 +205,7 @@ int tfsEncodeFile(void **buf, TFILE *pf) {
 
 void *tfsDecodeFile(void *buf, TFILE *pf) {
   int32_t level, id;
-  char *  rname;
+  char   *rname;
 
   buf = taosDecodeVariantI32(buf, &(level));
   buf = taosDecodeVariantI32(buf, &(id));
@@ -247,7 +232,7 @@ void tfsdirname(const TFILE *pf, char *dest) {
 }
 
 // DIR APIs ====================================
-int tfsMkdirAt(const char *rname, int level, int id) {
+int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id) {
   SDisk *pDisk = TFS_DISK_AT(level, id);
   char   aname[TMPNAME_LEN];
 
@@ -260,7 +245,7 @@ int tfsMkdirAt(const char *rname, int level, int id) {
   return 0;
 }
 
-int tfsMkdirRecurAt(const char *rname, int level, int id) {
+int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) {
   if (tfsMkdirAt(rname, level, id) < 0) {
     if (errno == ENOENT) {
       // Try to create upper
@@ -293,10 +278,10 @@ int tfsMkdirRecurAt(const char *rname, int level, int id) {
   return 0;
 }
 
-int tfsMkdir(const char *rname) {
-  for (int level = 0; level < TFS_NLEVEL(); level++) {
+int32_t tfsMkdir(const char *rname) {
+  for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
     STier *pTier = TFS_TIER_AT(level);
-    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+    for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) {
       if (tfsMkdirAt(rname, level, id) < 0) {
         return -1;
       }
@@ -306,15 +291,15 @@ int tfsMkdir(const char *rname) {
   return 0;
 }
 
-int tfsRmdir(const char *rname) {
+int32_t tfsRmdir(const char *rname) {
   char aname[TMPNAME_LEN] = "\0";
 
-  for (int level = 0; level < TFS_NLEVEL(); level++) {
+  for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
     STier *pTier = TFS_TIER_AT(level);
-    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
-      SDisk *pDisk = DISK_AT_TIER(pTier, id);
+    for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) {
+      SDisk *pDisk = pTier->disks[id];
 
-      snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
+      snprintf(aname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TS_PATH_DELIMITER, rname);
 
       taosRemoveDir(aname);
     }
@@ -323,13 +308,14 @@ int tfsRmdir(const char *rname) {
   return 0;
 }
 
-int tfsRename(char *orname, char *nrname) {
+#if 0
+int32_t tfsRename(char *orname, char *nrname) {
   char oaname[TMPNAME_LEN] = "\0";
   char naname[TMPNAME_LEN] = "\0";
 
-  for (int level = 0; level < pfs->nlevel; level++) {
+  for (int32_t level = 0; level < pfs->nlevel; level++) {
     STier *pTier = TFS_TIER_AT(level);
-    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+    for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) {
       SDisk *pDisk = DISK_AT_TIER(pTier, id);
 
       snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname);
@@ -341,20 +327,21 @@ int tfsRename(char *orname, char *nrname) {
 
   return 0;
 }
+#endif
 
 struct TDIR {
   SDiskIter iter;
-  int       level;
-  int       id;
+  int32_t   level;
+  int32_t   id;
   char      dirname[TSDB_FILENAME_LEN];
   TFILE     tfile;
-  DIR *     dir;
+  DIR      *dir;
 };
 
 TDIR *tfsOpendir(const char *rname) {
   TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir));
   if (tdir == NULL) {
-    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -407,9 +394,9 @@ void tfsClosedir(TDIR *tdir) {
 }
 
 // private
-static int tfsMount(SDiskCfg *pCfg) {
+static int32_t tfsMount(SDiskCfg *pCfg) {
   SDiskID did;
-  SDisk * pDisk = NULL;
+  SDisk  *pDisk = NULL;
 
   if (tfsCheckAndFormatCfg(pCfg) < 0) return -1;
 
@@ -419,7 +406,7 @@ static int tfsMount(SDiskCfg *pCfg) {
     fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
     return -1;
   }
-  did.id = DISK_ID(pDisk);
+  did.id = pDisk->id;
 
   taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did));
   if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1;
@@ -427,7 +414,7 @@ static int tfsMount(SDiskCfg *pCfg) {
   return 0;
 }
 
-static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
+static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
   char        dirName[TSDB_FILENAME_LEN] = "\0";
   struct stat pstat;
 
@@ -486,10 +473,10 @@ static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
   return 0;
 }
 
-static int tfsFormatDir(char *idir, char *odir) {
+static int32_t tfsFormatDir(char *idir, char *odir) {
   wordexp_t wep = {0};
 
-  int code = wordexp(idir, &wep, 0);
+  int32_t code = wordexp(idir, &wep, 0);
   if (code != 0) {
     terrno = TAOS_SYSTEM_ERROR(code);
     return -1;
@@ -507,14 +494,14 @@ static int tfsFormatDir(char *idir, char *odir) {
   return 0;
 }
 
-static int tfsCheck() {
+static int32_t tfsCheck() {
   if (TFS_PRIMARY_DISK() == NULL) {
     fError("no primary disk is set");
     terrno = TSDB_CODE_FS_NO_PRIMARY_DISK;
     return -1;
   }
 
-  for (int level = 0; level < TFS_NLEVEL(); level++) {
+  for (int32_t level = 0; level < TFS_NLEVEL(); level++) {
     if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) {
       fError("no disk at level %d", level);
       terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER;
@@ -528,8 +515,8 @@ static int tfsCheck() {
 static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); }
 static SDisk *tfsGetDiskByName(const char *dir) {
   SDiskID did;
-  SDisk * pDisk = NULL;
-  void *  pr = NULL;
+  SDisk  *pDisk = NULL;
+  void   *pr = NULL;
 
   pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN));
   if (pr == NULL) return NULL;
@@ -541,7 +528,7 @@ static SDisk *tfsGetDiskByName(const char *dir) {
   return pDisk;
 }
 
-static int tfsOpendirImpl(TDIR *tdir) {
+static int32_t tfsOpendirImpl(TDIR *tdir) {
   SDisk *pDisk = NULL;
   char   adir[TMPNAME_LEN * 2] = "\0";
 
@@ -554,10 +541,10 @@ static int tfsOpendirImpl(TDIR *tdir) {
     pDisk = tfsNextDisk(&(tdir->iter));
     if (pDisk == NULL) return 0;
 
-    tdir->level = DISK_LEVEL(pDisk);
-    tdir->id = DISK_ID(pDisk);
+    tdir->level = pDisk->level;
+    tdir->id = pDisk->id;
 
-    snprintf(adir, TMPNAME_LEN * 2, "%s/%s", DISK_DIR(pDisk), tdir->dirname);
+    snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TS_PATH_DELIMITER,tdir->dirname);
     tdir->dir = opendir(adir);
     if (tdir->dir != NULL) break;
   }
@@ -572,8 +559,8 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) {
 
   if (pDisk == NULL) return NULL;
 
-  int level = DISK_LEVEL(pDisk);
-  int id = DISK_ID(pDisk);
+  int32_t level = pDisk->level;
+  int32_t id = pDisk->id;
 
   id++;
   if (id < TIER_NDISKS(TFS_TIER_AT(level))) {
@@ -596,21 +583,21 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) {
 // OTHER FUNCTIONS ===================================
 void taosGetDisk() {
   const double unit = 1024 * 1024 * 1024;
-  SysDiskSize  diskSize;
+  SDiskSize    diskSize;
   SFSMeta      fsMeta;
 
-  tfsUpdateInfo(&fsMeta, NULL, 0);
-  tsTotalDataDirGB = (float)(fsMeta.tsize / unit);
+  tfsUpdateSize(&fsMeta);
+  tsTotalDataDirGB = (float)(fsMeta.total / unit);
   tsUsedDataDirGB = (float)(fsMeta.used / unit);
   tsAvailDataDirGB = (float)(fsMeta.avail / unit);
 
   if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
-    tsTotalLogDirGB = (float)(diskSize.tsize / unit);
+    tsTotalLogDirGB = (float)(diskSize.total / unit);
     tsAvailLogDirGB = (float)(diskSize.avail / unit);
   }
 
   if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
-    tsTotalTmpDirGB = (float)(diskSize.tsize / unit);
+    tsTotalTmpDirGB = (float)(diskSize.total / unit);
     tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
   }
 }

source/libs/tfs/src/tdisk.c → source/libs/tfs/src/tfsDisk.c

@@ -12,48 +12,45 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include "os.h"
 
-#include "taoserror.h"
-#include "tfsint.h"
+#define _DEFAULT_SOURCE
+#include "tfsInt.h"
 
-// PROTECTED ====================================
-SDisk *tfsNewDisk(int level, int id, const char *dir) {
-  SDisk *pDisk = (SDisk *)calloc(1, sizeof(*pDisk));
+SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) {
+  SDisk *pDisk = calloc(1, sizeof(SDisk));
   if (pDisk == NULL) {
-    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pDisk->path = strdup(path);
+  if (pDisk->path == NULL) {
+    free(pDisk);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
   pDisk->level = level;
   pDisk->id = id;
-  tstrncpy(pDisk->dir, dir, TSDB_FILENAME_LEN);
-
+  taosGetDiskSize(pDisk->path, &pDisk->size);
   return pDisk;
 }
 
 SDisk *tfsFreeDisk(SDisk *pDisk) {
-  if (pDisk) {
+  if (pDisk != NULL) {
+    free(pDisk->path);
     free(pDisk);
   }
+
   return NULL;
 }
 
-int tfsUpdateDiskInfo(SDisk *pDisk) {
-  ASSERT(pDisk != NULL);
-
-  SysDiskSize diskSize = {0};
-
-  int code = taosGetDiskSize(pDisk->dir, &diskSize);
-  if (code != 0) {
-    fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir,
-           strerror(errno));
+int32_t tfsUpdateDiskSize(SDisk *pDisk) {
+  if (taosGetDiskSize(pDisk->path, &pDisk->size) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
+    fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr());
+    return -1;
   }
 
-  pDisk->dmeta.size = diskSize.tsize;
-  pDisk->dmeta.used = diskSize.used;
-  pDisk->dmeta.free = diskSize.avail;
-
-  return code;
+  return 0;
 }

source/libs/tfs/src/ttier.c → source/libs/tfs/src/tfsTier.c

@@ -12,53 +12,43 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include "os.h"
 
-#include "taosdef.h"
-#include "taoserror.h"
-#include "tfsint.h"
+#define _DEFAULT_SOURCE
+#include "tfsInt.h"
 
-#define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock))
-#define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock))
+#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock)
+#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock)
 
-// PROTECTED ==========================================
-int tfsInitTier(STier *pTier, int level) {
-  memset((void *)pTier, 0, sizeof(*pTier));
+int32_t tfsInitTier(STier *pTier, int32_t level) {
+  memset(pTier, 0, sizeof(STier));
 
-  int code = pthread_spin_init(&(pTier->lock), 0);
-  if (code) {
-    terrno = TAOS_SYSTEM_ERROR(code);
+  if (pthread_spin_init(&pTier->lock, 0) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
   pTier->level = level;
-
   return 0;
 }
 
 void tfsDestroyTier(STier *pTier) {
-  for (int id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) {
-    DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id));
+  for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) {
+    pTier->disks[id] = tfsFreeDisk(pTier->disks[id]);
   }
 
   pTier->ndisk = 0;
-
   pthread_spin_destroy(&(pTier->lock));
 }
 
 SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
-  ASSERT(pTier->level == pCfg->level);
-
-  int    id = 0;
-  SDisk *pDisk;
-
-  if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) {
+  if (pTier->ndisk >= TSDB_MAX_DISKS_PER_TIER) {
     terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
     return NULL;
   }
 
+  int32_t id = 0;
   if (pTier->level == 0) {
-    if (DISK_AT_TIER(pTier, 0) != NULL) {
+    if (pTier->disks[0] != NULL) {
       id = pTier->ndisk;
     } else {
       if (pCfg->primary) {
@@ -66,102 +56,85 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
       } else {
         id = pTier->ndisk + 1;
       }
-      if (id >= TSDB_MAX_DISKS_PER_TIER) {
-        terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
-        return NULL;
-      }
     }
   } else {
     id = pTier->ndisk;
   }
 
-  pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
+  if (id >= TSDB_MAX_DISKS_PER_TIER) {
+    terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
+    return NULL;
+  }
+
+  SDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
   if (pDisk == NULL) return NULL;
-  DISK_AT_TIER(pTier, id) = pDisk;
+
+  pTier->disks[id] = pDisk;
   pTier->ndisk++;
 
   fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id);
-
-  return DISK_AT_TIER(pTier, id);
+  return pTier->disks[id];
 }
 
-void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) {
-  STierMeta tmeta;
-
-  if (pTierMeta == NULL) {
-    pTierMeta = &tmeta;
-  }
-  memset(pTierMeta, 0, sizeof(*pTierMeta));
+void tfsUpdateTierSize(STier *pTier) {
+  SDiskSize size = {0};
+  int16_t   nAvailDisks = 0;
 
   tfsLockTier(pTier);
 
-  for (int id = 0; id < pTier->ndisk; id++) {
-    if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) < 0) {
-      continue;
-    }
-    pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id));
-    pTierMeta->used += DISK_USED_SIZE(DISK_AT_TIER(pTier, id));
-    pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id));
-    pTierMeta->nAvailDisks++;
+  for (int32_t id = 0; id < pTier->ndisk; id++) {
+    SDisk *pDisk = pTier->disks[id];
+    if (pDisk == NULL) continue;
+
+    size.total += pDisk->size.total;
+    size.used += pDisk->size.used;
+    size.avail += pDisk->size.avail;
+    nAvailDisks++;
   }
 
-  pTier->tmeta = *pTierMeta;
+  pTier->size = size;
+  pTier->nAvailDisks = nAvailDisks;
 
   tfsUnLockTier(pTier);
 }
 
 // Round-Robin to allocate disk on a tier
-int tfsAllocDiskOnTier(STier *pTier) {
-  ASSERT(pTier->ndisk > 0);
-  int    id = TFS_UNDECIDED_ID;
-  SDisk *pDisk;
+int32_t tfsAllocDiskOnTier(STier *pTier) {
+  terrno = TSDB_CODE_FS_NO_VALID_DISK;
 
   tfsLockTier(pTier);
 
-  if (TIER_AVAIL_DISKS(pTier) <= 0) {
+  if (pTier->ndisk <= 0 || pTier->nAvailDisks <= 0) {
     tfsUnLockTier(pTier);
-    return id;
+    return -1;
   }
 
-  id = pTier->nextid;
-  while (true) {
-    pDisk = DISK_AT_TIER(pTier, id);
-    ASSERT(pDisk != NULL);
+  int32_t retId = -1;
+  for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; ++id) {
+    int32_t diskId = (pTier->nextid + id) % pTier->ndisk;
+    SDisk  *pDisk = pTier->disks[diskId];
 
-    if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) {
-      id = (id + 1) % pTier->ndisk;
-      if (id == pTier->nextid) {
-        tfsUnLockTier(pTier);
-        return TFS_UNDECIDED_ID;
-      } else {
-        continue;
-      }
-    } else {
-      pTier->nextid = (id + 1) % pTier->ndisk;
-      break;
-    }
-  }
+    if (pDisk == NULL) continue;
 
-  tfsUnLockTier(pTier);
-  return id;
-}
+    if (pDisk->size.avail < TFS_MIN_DISK_FREE_SIZE) continue;
 
-void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) {
-  ASSERT(pTierMeta != NULL);
+    retId = diskId;
+    terrno = 0;
+    pTier->nextid = (diskId + 1) % pTier->ndisk;
+    break;
+  }
 
-  tfsLockTier(pTier);
-  *pTierMeta = pTier->tmeta;
   tfsUnLockTier(pTier);
+  return retId;
 }
 
 void tfsPosNextId(STier *pTier) {
-  ASSERT(pTier->ndisk > 0);
-  int nextid = 0;
+  int32_t nextid = 0;
 
-  for (int id = 1; id < pTier->ndisk; id++) {
-    SDisk *pLDisk = DISK_AT_TIER(pTier, nextid);
-    SDisk *pDisk = DISK_AT_TIER(pTier, id);
-    if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) {
+  for (int32_t id = 1; id < pTier->ndisk; id++) {
+    SDisk *pLDisk = pTier->disks[nextid];
+    SDisk *pDisk = pTier->disks[id];
+    if (pDisk->size.avail > TFS_MIN_DISK_FREE_SIZE && pDisk->size.avail > pLDisk->size.avail) {
       nextid = id;
     }
   }

source/os/src/osSysinfo.c

@@ -121,7 +121,7 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
+int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
   unsigned _int64 i64FreeBytesToCaller;
   unsigned _int64 i64TotalBytes;
   unsigned _int64 i64FreeBytes;
@@ -438,7 +438,7 @@ int taosSystem(const char *cmd) {
 
 void taosSetCoreDump() {}
 
-int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
+int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
   struct statvfs info;
   if (statvfs(dataDir, &info)) {
     //printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
@@ -771,13 +771,12 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
+int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
   struct statvfs info;
   if (statvfs(dataDir, &info)) {
-    //printf("failed to get disk size, dataDir:%s errno:%s", dataDir, strerror(errno));
     return -1;
   } else {
-    diskSize->tsize = info.f_blocks * info.f_frsize;
+    diskSize->total = info.f_blocks * info.f_frsize;
     diskSize->avail = info.f_bavail * info.f_frsize;
     diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize;
     return 0;

source/util/src/terror.c

@@ -396,7 +396,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED,           "WAL file is corrupted
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT,               "WAL size exceeds limit")
 
 // tfs
-TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY,             "tfs out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR,                "tfs out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG,                 "tfs invalid mount config")
 TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT,            "tfs too many mount")
 TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY,               "tfs duplicate primary mount")

src/inc/tfs.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TD_TFS_H
-#define TD_TFS_H
-
-#include "tglobal.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
-  int level;
-  int id;
-} SDiskID;
-
-#define TFS_UNDECIDED_LEVEL -1
-#define TFS_UNDECIDED_ID -1
-#define TFS_PRIMARY_LEVEL 0
-#define TFS_PRIMARY_ID 0
-#define TFS_MIN_LEVEL 0
-#define TFS_MAX_LEVEL (TSDB_MAX_TIERS - 1)
-
-// FS APIs ====================================
-typedef struct {
-  int64_t tsize;
-  int64_t used;
-  int64_t avail;
-} SFSMeta;
-
-typedef struct {
-  int64_t size;
-  int64_t used;
-  int64_t free;
-  int16_t nAvailDisks;  // # of Available disks
-} STierMeta;
-
-int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
-void tfsCleanup();
-void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels);
-void tfsGetMeta(SFSMeta *pMeta);
-void tfsAllocDisk(int expLevel, int *level, int *id);
-
-const char *TFS_PRIMARY_PATH();
-const char *TFS_DISK_PATH(int level, int id);
-
-// TFILE APIs ====================================
-typedef struct {
-  int  level;
-  int  id;
-  char rname[TSDB_FILENAME_LEN];  // REL name
-  char aname[TSDB_FILENAME_LEN];  // ABS name
-} TFILE;
-
-#define TFILE_LEVEL(pf) ((pf)->level)
-#define TFILE_ID(pf) ((pf)->id)
-#define TFILE_NAME(pf) ((pf)->aname)
-#define TFILE_REL_NAME(pf) ((pf)->rname)
-
-#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
-#define tfsclose(fd) close(fd)
-#define tfsremove(pf) remove(TFILE_NAME(pf))
-#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df))
-#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df))
-
-void  tfsInitFile(TFILE *pf, int level, int id, const char *bname);
-bool  tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
-int   tfsEncodeFile(void **buf, TFILE *pf);
-void *tfsDecodeFile(void *buf, TFILE *pf);
-void  tfsbasename(const TFILE *pf, char *dest);
-void  tfsdirname(const TFILE *pf, char *dest);
-
-// DIR APIs ====================================
-int tfsMkdirAt(const char *rname, int level, int id);
-int tfsMkdirRecurAt(const char *rname, int level, int id);
-int tfsMkdir(const char *rname);
-int tfsRmdir(const char *rname);
-int tfsRename(char *orname, char *nrname);
-
-typedef struct TDIR TDIR;
-
-TDIR *       tfsOpendir(const char *rname);
-const TFILE *tfsReaddir(TDIR *tdir);
-void         tfsClosedir(TDIR *tdir);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif