refact tfs module

include/libs/tfs/tfs.h

@@ -16,77 +16,226 @@
 #ifndef _TD_TFS_H_
 #define _TD_TFS_H_
 
-#include "tglobal.h"
+#include "tcfg.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+/* ------------------------ TYPES EXPOSED ------------------------ */
+typedef struct STfs STfs;
+typedef struct STfsDir STfsDir;
+
 typedef struct {
   int32_t level;
   int32_t id;
 } SDiskID;
 
-#define TFS_UNDECIDED_LEVEL -1
-#define TFS_UNDECIDED_ID -1
-#define TFS_PRIMARY_LEVEL 0
-#define TFS_PRIMARY_ID 0
-#define TFS_MIN_LEVEL 0
-#define TFS_MAX_LEVEL (TSDB_MAX_TIERS - 1)
-
-// FS APIs ====================================
 typedef struct {
-  int64_t total;
-  int64_t used;
-  int64_t avail;
-} SFSMeta;
+  SDiskID did;
+  char    aname[TSDB_FILENAME_LEN];  // ABS name
+  char    rname[TSDB_FILENAME_LEN];  // REL name
+  STfs   *pTfs;
+} STfsFile;
 
-int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk);
-void    tfsCleanup();
-void    tfsUpdateSize(SFSMeta *pFSMeta);
-void    tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id);
+/**
+ * @brief Open a fs.
+ *
+ * @param pCfg Config of the fs.
+ * @param ndisk Length of the config.
+ * @return STfs* The fs object.
+ */
+STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk);
 
-const char *TFS_PRIMARY_PATH();
-const char *TFS_DISK_PATH(int32_t level, int32_t id);
+/**
+ * @brief Close a fs.
+ *
+ * @param pTfs The fs object to close.
+ */
+void tfsClose(STfs *pTfs);
 
-// TFILE APIs ====================================
-typedef struct {
-  int32_t level;
-  int32_t id;
-  char    rname[TSDB_FILENAME_LEN];  // REL name
-  char    aname[TSDB_FILENAME_LEN];  // ABS name
-} TFILE;
-
-#define TFILE_LEVEL(pf) ((pf)->level)
-#define TFILE_ID(pf) ((pf)->id)
-#define TFILE_NAME(pf) ((pf)->aname)
-#define TFILE_REL_NAME(pf) ((pf)->rname)
-
-#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
-#define tfsclose(fd) close(fd)
-#define tfsremove(pf) remove(TFILE_NAME(pf))
-#define tfscopy(sf, df) taosCopyFile(TFILE_NAME(sf), TFILE_NAME(df))
-#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df))
-
-void    tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname);
-bool    tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
-int32_t tfsEncodeFile(void **buf, TFILE *pf);
-void   *tfsDecodeFile(void *buf, TFILE *pf);
-void    tfsbasename(const TFILE *pf, char *dest);
-void    tfsdirname(const TFILE *pf, char *dest);
-
-// DIR APIs ====================================
-int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id);
-int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id);
-int32_t tfsMkdir(const char *rname);
-int32_t tfsRmdir(const char *rname);
-int32_t tfsRename(char *orname, char *nrname);
-
-typedef struct TDIR TDIR;
-
-TDIR        *tfsOpendir(const char *rname);
-const TFILE *tfsReaddir(TDIR *tdir);
-void         tfsClosedir(TDIR *tdir);
+/**
+ * @brief Update the disk size.
+ *
+ * @param pTfs The fs object.
+ */
+void tfsUpdateSize(STfs *pTfs);
+
+/**
+ * @brief Get the disk size.
+ *
+ * @param pTfs The fs object.
+ */
+SDiskSize tfsGetSize(STfs *pTfs);
+
+/**
+ * @brief Allocate an existing available tier level from fs.
+ *
+ * @param pTfs The fs object.
+ * @param expLevel Disk level want to allocate.
+ * @param pDiskId The disk ID after allocation.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId);
+
+/**
+ * @brief Get the primary path.
+ *
+ * @param pTfs The fs object.
+ * @return const char * The primary path.
+ */
+const char *tfsGetPrimaryPath(STfs *pTfs);
+
+/**
+ * @brief Get the disk path.
+ *
+ * @param pTfs The fs object.
+ * @param diskId The diskId.
+ * @return const char * The primary path.
+ */
+const char *tfsGetDiskPath(STfs *pTfs, SDiskID diskId);
+
+/**
+ * @brief Make directory at all levels in tfs.
+ *
+ * @param pTfs The fs object.
+ * @param rname The rel name of directory.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsMkdir(STfs *pTfs, const char *rname);
+
+/**
+ * @brief Create directories in tfs.
+ *
+ * @param pTfs The fs object.
+ * @param rname The rel name of directory.
+ * @param diskId The disk ID.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId);
+
+/**
+ * @brief Recursive create directories in tfs.
+ *
+ * @param pTfs The fs object.
+ * @param rname The rel name of directory.
+ * @param diskId The disk ID.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId);
+
+/**
+ * @brief Remove directory at all levels in tfs.
+ *
+ * @param pTfs The fs object.
+ * @param rname The rel name of directory.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsRmdir(STfs *pTfs, const char *rname);
+
+/**
+ * @brief Rename file/directory in tfs.
+ *
+ * @param pTfs The fs object.
+ * @param orname The rel name of old file.
+ * @param nrname The rel name of new file.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsRename(STfs *pTfs, char *orname, char *nrname);
+
+/**
+ * @brief Init file object in tfs.
+ *
+ * @param pTfs The fs object.
+ * @param pFile The file object.
+ * @param diskId The disk ID.
+ * @param rname The rel name of file.
+ */
+void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname);
+
+/**
+ * @brief Determine whether they are the same file.
+ *
+ * @param pFile1 The file object.
+ * @param pFile2 The file object.
+ * @param bool The compare result.
+ */
+bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2);
+
+/**
+ * @brief Encode file name to a buffer.
+ *
+ * @param buf The buffer where file name are saved.
+ * @param pFile The file object.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsEncodeFile(void **buf, STfsFile *pFile);
+
+/**
+ * @brief Decode file name from a buffer.
+ *
+ * @param pTfs The fs object.
+ * @param buf The buffer where file name are saved.
+ * @param pFile The file object.
+ * @return void * Buffer address after decode.
+ */
+void *tfsDecodeFile(STfs *pTfs, void *buf, STfsFile *pFile);
+
+/**
+ * @brief Get the basename of the file.
+ *
+ * @param pFile The file object.
+ * @param dest The buffer where basename will be saved.
+ */
+void tfsBasename(const STfsFile *pFile, char *dest);
+
+/**
+ * @brief Get the dirname of the file.
+ *
+ * @param pFile The file object.
+ * @param dest The buffer where dirname will be saved.
+ */
+void tfsDirname(const STfsFile *pFile, char *dest);
+
+/**
+ * @brief Remove file in tfs.
+ *
+ * @param pFile The file to be removed.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsRemoveFile(const STfsFile *pFile);
+
+/**
+ * @brief Copy file in tfs.
+ *
+ * @param pFile1 The src file.
+ * @param pFile2 The dest file.
+ * @return int32_t 0 for success, -1 for failure.
+ */
+int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2);
+
+/**
+ * @brief Open a directory for traversal.
+ *
+ * @param rname The rel name of file.
+ * @return STfsDir* The dir object.
+ */
+STfsDir *tfsOpendir(STfs *pTfs, const char *rname);
+
+/**
+ * @brief Get a file from dir and move to next pos.
+ *
+ * @param pDir The dir object.
+ * @return STfsFile* The file in dir.
+ */
+const STfsFile *tfsReaddir(STfsDir *pDir);
+
+/**
+ * @brief Close a directory.
+ *
+ * @param pDir The dir object.
+ */
+void tfsClosedir(STfsDir *pDir);
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -372,10 +372,14 @@ do { \
 
 #define TSDB_ARB_DUMMY_TIME                    4765104000000 // 2121-01-01 00:00:00.000, :P
 
-#define TSDB_MAX_TIERS           3
-#define TSDB_MAX_DISKS_PER_TIER 16
-#define TSDB_MAX_DISKS           (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER)
-
+#define TFS_MAX_TIERS 3
+#define TFS_MAX_DISKS_PER_TIER 16
+#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER)
+#define TFS_MIN_LEVEL 0
+#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1)
+#define TFS_PRIMARY_LEVEL 0
+#define TFS_PRIMARY_ID 0
+#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
 
 enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED };
 enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK };

source/common/src/tglobal.c

@@ -137,7 +137,7 @@ int32_t tsDiskCfgNum = 0;
 #ifndef _STORAGE
 SDiskCfg tsDiskCfg[1];
 #else
-SDiskCfg tsDiskCfg[TSDB_MAX_DISKS];
+SDiskCfg tsDiskCfg[TFS_MAX_DISKS];
 #endif
 
 /*

source/dnode/mgmt/impl/inc/dndEnv.h

@@ -134,6 +134,7 @@ typedef struct SDnode {
   SBnodeMgmt   bmgmt;
   SVnodesMgmt  vmgmt;
   STransMgmt   tmgmt;
+  STfs        *pTfs;
   SStartupReq  startup;
 } SDnode;
 

source/dnode/mgmt/impl/inc/dndInt.h

@@ -43,6 +43,7 @@ extern "C" {
 #include "qnode.h"
 #include "snode.h"
 #include "vnode.h"
+#include "tfs.h"
 
 extern int32_t dDebugFlag;
 

source/dnode/mgmt/impl/src/dndEnv.c

@@ -173,11 +173,12 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) {
     return NULL;
   }
 
-  SDiskCfg dCfg;
-  strcpy(dCfg.dir, pDnode->cfg.dataDir);
+  SDiskCfg dCfg = {0};
+  tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN);
   dCfg.level = 0;
   dCfg.primary = 1;
-  if (tfsInit(&dCfg, 1) != 0) {
+  pDnode->pTfs = tfsOpen(&dCfg, 1);
+  if (pDnode->pTfs == NULL) {
     dError("failed to init tfs since %s", terrstr());
     dndClose(pDnode);
     return NULL;
@@ -251,7 +252,7 @@ void dndClose(SDnode *pDnode) {
   dndCleanupQnode(pDnode);
   dndCleanupVnodes(pDnode);
   dndCleanupMgmt(pDnode);
-  tfsCleanup();
+  tfsClose(pDnode->pTfs);
 
   dndCloseImp(pDnode);
   free(pDnode);
@@ -313,4 +314,28 @@ void dndCleanup() {
 
   taosStopCacheRefreshWorker();
   dInfo("dnode env is cleaned up");
+}
+
+// OTHER FUNCTIONS ===================================
+void taosGetDisk() {
+#if 0  
+  const double unit = 1024 * 1024 * 1024;
+  
+  SDiskSize    diskSize = tfsGetSize(pTfs);
+  
+  tfsUpdateSize(&fsMeta);
+  tsTotalDataDirGB = (float)(fsMeta.total / unit);
+  tsUsedDataDirGB = (float)(fsMeta.used / unit);
+  tsAvailDataDirGB = (float)(fsMeta.avail / unit);
+
+  if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
+    tsTotalLogDirGB = (float)(diskSize.total / unit);
+    tsAvailLogDirGB = (float)(diskSize.avail / unit);
+  }
+
+  if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
+    tsTotalTmpDirGB = (float)(diskSize.total / unit);
+    tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
+  }
+#endif
 }
\ No newline at end of file

source/dnode/vnode/inc/tsdb.h

@@ -80,7 +80,7 @@ typedef struct {
 } STableKeyInfo;
 
 // STsdb
-STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta);
+STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs);
 void   tsdbClose(STsdb *);
 void   tsdbRemove(const char *path);
 int    tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp);

source/dnode/vnode/inc/vnode.h

@@ -36,7 +36,8 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq);
 
 typedef struct SVnodeCfg {
   int32_t  vgId;
-  SDnode * pDnode;
+  SDnode  *pDnode;
+  STfs    *pTfs;
   uint64_t wsize;
   uint64_t ssize;
   uint64_t lsize;

source/dnode/vnode/src/inc/tsdbDef.h

@@ -50,6 +50,7 @@ struct STsdb {
   SMemAllocatorFactory *pmaf;
   STsdbFS *             fs;
   SMeta *               pMeta;
+  STfs *                pTfs;
 };
 
 #define REPO_ID(r) ((r)->vgId)

source/dnode/vnode/src/inc/tsdbFile.h

@@ -54,10 +54,10 @@ typedef struct {
 } SMFInfo;
 
 typedef struct {
-  SMFInfo info;
-  TFILE   f;
-  int     fd;
-  uint8_t state;
+  SMFInfo  info;
+  STfsFile f;
+  int      fd;
+  uint8_t  state;
 } SMFile;
 
 void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
@@ -175,10 +175,10 @@ typedef struct {
 } SDFInfo;
 
 typedef struct {
-  SDFInfo info;
-  TFILE   f;
-  int     fd;
-  uint8_t state;
+  SDFInfo  info;
+  STfsFile f;
+  int      fd;
+  uint8_t  state;
 } SDFile;
 
 void  tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype);

source/dnode/vnode/src/inc/vnd.h

@@ -78,6 +78,7 @@ struct SVnode {
   tsem_t     canCommit;
   SQHandle*  pQuery;
   SDnode*    pDnode;
+  STfs*      pTfs;
 };
 
 int vnodeScheduleTask(SVnodeTask* task);

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -97,8 +97,7 @@ int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
 
   level = tsdbGetFidLevel(pSet->fid, pRtn);
 
-  tfsAllocDisk(level, &(did.level), &(did.id));
-  if (did.level == TFS_UNDECIDED_LEVEL) {
+  if (tfsAllocDisk(pRepo->pTfs, level, &did) < 0) {
     terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
     return -1;
   }
@@ -456,8 +455,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
   STsdb *    pRepo = TSDB_COMMIT_REPO(pCommith);
   SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith);
 
-  tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id));
-  if (did.level == TFS_UNDECIDED_LEVEL) {
+  if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) {
     terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
     return -1;
   }

source/dnode/vnode/src/tsdb/tsdbCompact.c

@@ -186,8 +186,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) {
       }
     } else {
       // Create new fset as compacted fset
-      tfsAllocDisk(tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &(did.level), &(did.id));
-      if (did.level == TFS_UNDECIDED_LEVEL) {
+      if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &did) < 0) {
         terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
         tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
         tsdbCompactFSetEnd(pComph);

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -23,14 +23,14 @@ static const char *tsdbTxnFname[] = {"current.t", "current"};
 
 static int  tsdbComparFidFSet(const void *arg1, const void *arg2);
 static void tsdbResetFSStatus(SFSStatus *pStatus);
-static int  tsdbSaveFSStatus(SFSStatus *pStatus, int vid);
+static int  tsdbSaveFSStatus(STfs *pTfs, SFSStatus *pStatus, int vid);
 static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo);
-static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]);
+static void tsdbGetTxnFname(STfs *pTfs, int repoid, TSDB_TXN_FILE_T ftype, char fname[]);
 static int  tsdbOpenFSFromCurrent(STsdb *pRepo);
 static int  tsdbScanAndTryFixFS(STsdb *pRepo);
 static int  tsdbScanRootDir(STsdb *pRepo);
 static int  tsdbScanDataDir(STsdb *pRepo);
-static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf);
+static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf);
 static int  tsdbRestoreCurrent(STsdb *pRepo);
 static int  tsdbComparTFILE(const void *arg1, const void *arg2);
 static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired);
@@ -311,7 +311,7 @@ int tsdbOpenFS(STsdb *pRepo) {
 
   ASSERT(pfs != NULL);
 
-  tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current);
+  tsdbGetTxnFname(pRepo->pTfs, REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current);
 
   tsdbGetRtnSnap(pRepo, &pRepo->rtn);
   if (access(current, F_OK) == 0) {
@@ -375,7 +375,7 @@ int tsdbEndFSTxn(STsdb *pRepo) {
   SFSStatus *pStatus;
 
   // Write current file system snapshot
-  if (tsdbSaveFSStatus(pfs->nstatus, REPO_ID(pRepo)) < 0) {
+  if (tsdbSaveFSStatus(pRepo->pTfs, pfs->nstatus, REPO_ID(pRepo)) < 0) {
     tsdbEndFSTxnWithError(pfs);
     return -1;
   }
@@ -405,7 +405,7 @@ int tsdbEndFSTxnWithError(STsdbFS *pfs) {
 
 int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); }
 
-static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
+static int tsdbSaveFSStatus(STfs *pTfs, SFSStatus *pStatus, int vid) {
   SFSHeader fsheader;
   void *    pBuf = NULL;
   void *    ptr;
@@ -413,8 +413,8 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
   char      tfname[TSDB_FILENAME_LEN] = "\0";
   char      cfname[TSDB_FILENAME_LEN] = "\0";
 
-  tsdbGetTxnFname(vid, TSDB_TXN_TEMP_FILE, tfname);
-  tsdbGetTxnFname(vid, TSDB_TXN_CURR_FILE, cfname);
+  tsdbGetTxnFname(pTfs, vid, TSDB_TXN_TEMP_FILE, tfname);
+  tsdbGetTxnFname(pTfs, vid, TSDB_TXN_CURR_FILE, cfname);
 
   int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
   if (fd < 0) {
@@ -645,8 +645,8 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
   }
 }
 
-static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]) {
-  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", TFS_PRIMARY_PATH(), repoid, tsdbTxnFname[ftype]);
+static void tsdbGetTxnFname(STfs *pTfs, int repoid, TSDB_TXN_FILE_T ftype, char fname[]) {
+  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(pTfs), repoid, tsdbTxnFname[ftype]);
 }
 
 static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
@@ -657,7 +657,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
   char      current[TSDB_FILENAME_LEN] = "\0";
   void *    ptr;
 
-  tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current);
+  tsdbGetTxnFname(pRepo->pTfs, REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current);
 
   // current file exists, try to recover
   fd = open(current, O_RDONLY | O_BINARY);
@@ -910,17 +910,17 @@ static int tsdbScanRootDir(STsdb *pRepo) {
   char         rootDir[TSDB_FILENAME_LEN];
   char         bname[TSDB_FILENAME_LEN];
   STsdbFS *    pfs = REPO_FS(pRepo);
-  const TFILE *pf;
+  const STfsFile *pf;
 
   tsdbGetRootDir(REPO_ID(pRepo), rootDir);
-  TDIR *tdir = tfsOpendir(rootDir);
+  STfsDir *tdir = tfsOpendir(rootDir);
   if (tdir == NULL) {
     tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
     return -1;
   }
 
   while ((pf = tfsReaddir(tdir))) {
-    tfsbasename(pf, bname);
+    tfsBasename(pf, bname);
 
     if (strcmp(bname, tsdbTxnFname[TSDB_TXN_CURR_FILE]) == 0 || strcmp(bname, "data") == 0) {
       // Skip current file and data directory
@@ -944,17 +944,17 @@ static int tsdbScanDataDir(STsdb *pRepo) {
   char         dataDir[TSDB_FILENAME_LEN];
   char         bname[TSDB_FILENAME_LEN];
   STsdbFS *    pfs = REPO_FS(pRepo);
-  const TFILE *pf;
+  const STfsFile *pf;
 
   tsdbGetDataDir(REPO_ID(pRepo), dataDir);
-  TDIR *tdir = tfsOpendir(dataDir);
+  STfsDir *tdir = tfsOpendir(dataDir);
   if (tdir == NULL) {
     tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno));
     return -1;
   }
 
   while ((pf = tfsReaddir(tdir))) {
-    tfsbasename(pf, bname);
+    tfsBasename(pf, bname);
 
     if (!tsdbIsTFileInFS(pfs, pf)) {
       (void)tfsremove(pf);
@@ -967,7 +967,7 @@ static int tsdbScanDataDir(STsdb *pRepo) {
   return 0;
 }
 
-static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
+static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) {
   SFSIter fsiter;
   tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
   SDFileSet *pSet;
@@ -987,8 +987,8 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
 // static int tsdbRestoreMeta(STsdb *pRepo) {
 //   char         rootDir[TSDB_FILENAME_LEN];
 //   char         bname[TSDB_FILENAME_LEN];
-//   TDIR *       tdir = NULL;
-//   const TFILE *pf = NULL;
+//   STfsDir *       tdir = NULL;
+//   const STfsFile *pf = NULL;
 //   const char * pattern = "^meta(-ver[0-9]+)?$";
 //   regex_t      regex;
 //   STsdbFS *    pfs = REPO_FS(pRepo);
@@ -1007,7 +1007,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
 //   }
 
 //   while ((pf = tfsReaddir(tdir))) {
-//     tfsbasename(pf, bname);
+//     tfsBasename(pf, bname);
 
 //     if (strcmp(bname, "data") == 0) {
 //       // Skip the data/ directory
@@ -1108,8 +1108,8 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
 static int tsdbRestoreDFileSet(STsdb *pRepo) {
   char         dataDir[TSDB_FILENAME_LEN];
   char         bname[TSDB_FILENAME_LEN];
-  TDIR *       tdir = NULL;
-  const TFILE *pf = NULL;
+  STfsDir *       tdir = NULL;
+  const STfsFile *pf = NULL;
   const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last)(-ver[0-9]+)?$";
   SArray *     fArray = NULL;
   regex_t      regex;
@@ -1120,7 +1120,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
   // Resource allocation and init
   regcomp(&regex, pattern, REG_EXTENDED);
 
-  fArray = taosArrayInit(1024, sizeof(TFILE));
+  fArray = taosArrayInit(1024, sizeof(STfsFile));
   if (fArray == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir,
@@ -1139,7 +1139,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
   }
 
   while ((pf = tfsReaddir(tdir))) {
-    tfsbasename(pf, bname);
+    tfsBasename(pf, bname);
 
     int code = regexec(&regex, bname, 0, NULL, 0);
     if (code == 0) {
@@ -1200,7 +1200,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
       uint32_t    tversion;
       char        _bname[TSDB_FILENAME_LEN];
 
-      tfsbasename(pf, _bname);
+      tfsBasename(pf, _bname);
       tsdbParseDFilename(_bname, &tvid, &tfid, &ttype, &tversion);
 
       ASSERT(tvid == REPO_ID(pRepo));
@@ -1287,7 +1287,7 @@ static int tsdbRestoreCurrent(STsdb *pRepo) {
     return -1;
   }
 
-  if (tsdbSaveFSStatus(pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) {
+  if (tsdbSaveFSStatus(pRepo->pTfs, pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) {
     tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno));
     return -1;
   }
@@ -1296,8 +1296,8 @@ static int tsdbRestoreCurrent(STsdb *pRepo) {
 }
 
 static int tsdbComparTFILE(const void *arg1, const void *arg2) {
-  TFILE *pf1 = (TFILE *)arg1;
-  TFILE *pf2 = (TFILE *)arg2;
+  STfsFile *pf1 = (STfsFile *)arg1;
+  STfsFile *pf2 = (STfsFile *)arg2;
 
   int         vid1, fid1, vid2, fid2;
   TSDB_FILE_T ftype1, ftype2;
@@ -1305,8 +1305,8 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) {
   char        bname1[TSDB_FILENAME_LEN];
   char        bname2[TSDB_FILENAME_LEN];
 
-  tfsbasename(pf1, bname1);
-  tfsbasename(pf2, bname2);
+  tfsBasename(pf1, bname1);
+  tfsBasename(pf2, bname2);
   tsdbParseDFilename(bname1, &vid1, &fid1, &ftype1, &version1);
   tsdbParseDFilename(bname2, &vid2, &fid2, &ftype2, &version2);
 

source/dnode/vnode/src/tsdb/tsdbMain.c

@@ -16,12 +16,12 @@
 #include "tsdbDef.h"
 
 static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF,
-                      SMeta *pMeta);
+                      SMeta *pMeta, STfs *pTfs);
 static void   tsdbFree(STsdb *pTsdb);
 static int    tsdbOpenImpl(STsdb *pTsdb);
 static void   tsdbCloseImpl(STsdb *pTsdb);
 
-STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta) {
+STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs) {
   STsdb *pTsdb = NULL;
 
   // Set default TSDB Options
@@ -36,7 +36,7 @@ STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAl
   }
 
   // Create the handle
-  pTsdb = tsdbNew(path, vgId, pTsdbCfg, pMAF, pMeta);
+  pTsdb = tsdbNew(path, vgId, pTsdbCfg, pMAF, pMeta, pTfs);
   if (pTsdb == NULL) {
     // TODO: handle error
     return NULL;
@@ -64,7 +64,7 @@ void tsdbRemove(const char *path) { taosRemoveDir(path); }
 
 /* ------------------------ STATIC METHODS ------------------------ */
 static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF,
-                      SMeta *pMeta) {
+                      SMeta *pMeta, STfs *pTfs) {
   STsdb *pTsdb = NULL;
 
   pTsdb = (STsdb *)calloc(1, sizeof(STsdb));
@@ -78,6 +78,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
   tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg);
   pTsdb->pmaf = pMAF;
   pTsdb->pMeta = pMeta;
+  pTsdb->pTfs = pTfs;
 
   pTsdb->fs = tsdbNewFS(pTsdbCfg);
 
@@ -494,7 +495,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t
     }
   } else {  // get the named file at the specified index. If not there, return 0
     fname = malloc(256);
-    sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name);
+    sprintf(fname, "%s/vnode/vnode%d/%s", tfsGetPrimaryPath(pRepo->pTfs), REPO_ID(pRepo), name);
     if (access(fname, F_OK) != 0) {
       tfree(fname);
       return 0;

source/dnode/vnode/src/vnd/vnodeMain.c

@@ -75,6 +75,7 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) {
 
   pVnode->vgId = pVnodeCfg->vgId;
   pVnode->pDnode = pVnodeCfg->pDnode;
+  pVnode->pTfs = pVnodeCfg->pTfs;
   pVnode->path = strdup(path);
   vnodeOptionsCopy(&(pVnode->config), pVnodeCfg);
 
@@ -109,7 +110,7 @@ static int vnodeOpenImpl(SVnode *pVnode) {
 
   // Open tsdb
   sprintf(dir, "%s/tsdb", pVnode->path);
-  pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta);
+  pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs);
   if (pVnode->pTsdb == NULL) {
     // TODO: handle error
     return -1;

source/libs/tfs/inc/tfsInt.h

@@ -33,28 +33,73 @@ extern int32_t fsDebugFlag;
 #define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }}
 #define fWarn(...)  { if (fsDebugFlag & DEBUG_WARN)  { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }}
 #define fInfo(...)  { if (fsDebugFlag & DEBUG_INFO)  { taosPrintLog("TFS ", 255, __VA_ARGS__); }}
-#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }}
-#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }}
+#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }}
+#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }}
 
-// Global Definitions
-#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
-
-typedef struct SDisk {
+typedef struct {
   int32_t   level;
   int32_t   id;
   char     *path;
   SDiskSize size;
-} SDisk;
+} STfsDisk;
 
-typedef struct STier {
+typedef struct {
   pthread_spinlock_t lock;
   int32_t            level;
-  int16_t            nextid;       // next disk id to allocate
-  int16_t            ndisk;        // # of disks mounted to this tier
-  int16_t            nAvailDisks;  // # of Available disks
-  SDisk             *disks[TSDB_MAX_DISKS_PER_TIER];
+  int32_t            nextid;       // next disk id to allocate
+  int32_t            ndisk;        // # of disks mounted to this tier
+  int32_t            nAvailDisks;  // # of Available disks
+  STfsDisk          *disks[TFS_MAX_DISKS_PER_TIER];
+  SDiskSize          size;
+} STfsTier;
+
+typedef struct {
+  STfsDisk *pDisk;
+} SDiskIter;
+
+typedef struct STfsDir {
+  SDiskIter *iter;
+  SDiskID    did;
+  char       dirname[TSDB_FILENAME_LEN];
+  STfsFile   tfile;
+  DIR       *dir;
+  STfs      *pTfs;
+} STfsDir;
+
+typedef struct STfs {
+  pthread_spinlock_t lock;
   SDiskSize          size;
-} STier;
+  int32_t            nlevel;
+  STfsTier           tiers[TFS_MAX_TIERS];
+  SHashObj          *hash;  // name to did map
+} STfs;
+
+STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir);
+STfsDisk *tfsFreeDisk(STfsDisk *pDisk);
+int32_t   tfsUpdateDiskSize(STfsDisk *pDisk);
+
+int32_t   tfsInitTier(STfsTier *pTier, int32_t level);
+void      tfsDestroyTier(STfsTier *pTier);
+STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg);
+void      tfsUpdateTierSize(STfsTier *pTier);
+int32_t   tfsAllocDiskOnTier(STfsTier *pTier);
+void      tfsPosNextId(STfsTier *pTier);
+
+#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock)
+#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock)
+
+#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)
+
+#define tfsLock(pTfs) pthread_spin_lock(&(pTfs)->lock)
+#define tfsUnLock(pTfs) pthread_spin_unlock(&(pTfs)->lock)
+
+#define TFS_TIER_AT(pTfs, level) (&pTfs->tiers[level])
+#define TFS_DISK_AT(pTfs, did) (pTfs->tiers[(did).level].disks[(did).id])
+#define TFS_PRIMARY_DISK(pTfs) (pTfs->tiers[0].disks[0])
+
+#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL()))
+#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level))))
+#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id))
 
 #define TIER_LEVEL(pt) ((pt)->level)
 #define TIER_NDISKS(pt) ((pt)->ndisk)
@@ -64,17 +109,6 @@ typedef struct STier {
 #define DISK_AT_TIER(pt, id) ((pt)->disks[id])
 #define DISK_DIR(pd) ((pd)->path)
 
-SDisk  *tfsNewDisk(int32_t level, int32_t id, const char *dir);
-SDisk  *tfsFreeDisk(SDisk *pDisk);
-int32_t tfsUpdateDiskSize(SDisk *pDisk);
-
-int32_t tfsInitTier(STier *pTier, int32_t level);
-void    tfsDestroyTier(STier *pTier);
-SDisk  *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg);
-void    tfsUpdateTierSize(STier *pTier);
-int32_t tfsAllocDiskOnTier(STier *pTier);
-void    tfsPosNextId(STier *pTier);
-
 #ifdef __cplusplus
 }
 #endif

source/libs/tfs/src/tfsDisk.c

@@ -16,8 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "tfsInt.h"
 
-SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) {
-  SDisk *pDisk = calloc(1, sizeof(SDisk));
+STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) {
+  STfsDisk *pDisk = calloc(1, sizeof(STfsDisk));
   if (pDisk == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
@@ -36,7 +36,7 @@ SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) {
   return pDisk;
 }
 
-SDisk *tfsFreeDisk(SDisk *pDisk) {
+STfsDisk *tfsFreeDisk(STfsDisk *pDisk) {
   if (pDisk != NULL) {
     free(pDisk->path);
     free(pDisk);
@@ -45,7 +45,7 @@ SDisk *tfsFreeDisk(SDisk *pDisk) {
   return NULL;
 }
 
-int32_t tfsUpdateDiskSize(SDisk *pDisk) {
+int32_t tfsUpdateDiskSize(STfsDisk *pDisk) {
   if (taosGetDiskSize(pDisk->path, &pDisk->size) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr());

source/libs/tfs/src/tfsTier.c

@@ -16,11 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "tfsInt.h"
 
-#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock)
-#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock)
-
-int32_t tfsInitTier(STier *pTier, int32_t level) {
-  memset(pTier, 0, sizeof(STier));
+int32_t tfsInitTier(STfsTier *pTier, int32_t level) {
+  memset(pTier, 0, sizeof(STfsTier));
 
   if (pthread_spin_init(&pTier->lock, 0) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -31,17 +28,17 @@ int32_t tfsInitTier(STier *pTier, int32_t level) {
   return 0;
 }
 
-void tfsDestroyTier(STier *pTier) {
-  for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) {
+void tfsDestroyTier(STfsTier *pTier) {
+  for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; id++) {
     pTier->disks[id] = tfsFreeDisk(pTier->disks[id]);
   }
 
   pTier->ndisk = 0;
-  pthread_spin_destroy(&(pTier->lock));
+  pthread_spin_destroy(&pTier->lock);
 }
 
-SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
-  if (pTier->ndisk >= TSDB_MAX_DISKS_PER_TIER) {
+STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) {
+  if (pTier->ndisk >= TFS_MAX_DISKS_PER_TIER) {
     terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
     return NULL;
   }
@@ -61,12 +58,12 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
     id = pTier->ndisk;
   }
 
-  if (id >= TSDB_MAX_DISKS_PER_TIER) {
+  if (id >= TFS_MAX_DISKS_PER_TIER) {
     terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
     return NULL;
   }
 
-  SDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
+  STfsDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
   if (pDisk == NULL) return NULL;
 
   pTier->disks[id] = pDisk;
@@ -76,14 +73,14 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
   return pTier->disks[id];
 }
 
-void tfsUpdateTierSize(STier *pTier) {
+void tfsUpdateTierSize(STfsTier *pTier) {
   SDiskSize size = {0};
-  int16_t   nAvailDisks = 0;
+  int32_t   nAvailDisks = 0;
 
   tfsLockTier(pTier);
 
   for (int32_t id = 0; id < pTier->ndisk; id++) {
-    SDisk *pDisk = pTier->disks[id];
+    STfsDisk *pDisk = pTier->disks[id];
     if (pDisk == NULL) continue;
 
     size.total += pDisk->size.total;
@@ -99,7 +96,7 @@ void tfsUpdateTierSize(STier *pTier) {
 }
 
 // Round-Robin to allocate disk on a tier
-int32_t tfsAllocDiskOnTier(STier *pTier) {
+int32_t tfsAllocDiskOnTier(STfsTier *pTier) {
   terrno = TSDB_CODE_FS_NO_VALID_DISK;
 
   tfsLockTier(pTier);
@@ -110,9 +107,9 @@ int32_t tfsAllocDiskOnTier(STier *pTier) {
   }
 
   int32_t retId = -1;
-  for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; ++id) {
-    int32_t diskId = (pTier->nextid + id) % pTier->ndisk;
-    SDisk  *pDisk = pTier->disks[diskId];
+  for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; ++id) {
+    int32_t   diskId = (pTier->nextid + id) % pTier->ndisk;
+    STfsDisk *pDisk = pTier->disks[diskId];
 
     if (pDisk == NULL) continue;
 
@@ -128,12 +125,12 @@ int32_t tfsAllocDiskOnTier(STier *pTier) {
   return retId;
 }
 
-void tfsPosNextId(STier *pTier) {
+void tfsPosNextId(STfsTier *pTier) {
   int32_t nextid = 0;
 
   for (int32_t id = 1; id < pTier->ndisk; id++) {
-    SDisk *pLDisk = pTier->disks[nextid];
-    SDisk *pDisk = pTier->disks[id];
+    STfsDisk *pLDisk = pTier->disks[nextid];
+    STfsDisk *pDisk = pTier->disks[id];
     if (pDisk->size.avail > TFS_MIN_DISK_FREE_SIZE && pDisk->size.avail > pLDisk->size.avail) {
       nextid = id;
     }