Merge pull request #10328 from taosdata/feature/tdb

Feature/tdb

source/dnode/vnode/CMakeLists.txt

+set(META_DB_IMPL_LIST "BDB" "TDB")
+set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation")
+set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST})
+
+if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)
+    message(STATUS "META DB Impl: ${META_DB_IMPL}==============")
+else()
+    message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============")
+endif()
+
 aux_source_directory(src/meta META_SRC)
+if(${META_DB_IMPL} STREQUAL "BDB")
+    list(REMOVE_ITEM META_SRC "src/meta/metaTDBImpl.c")
+elseif(${META_DB_IMPL} STREQUAL "TDB")
+    list(REMOVE_ITEM META_SRC "src/meta/metaBDBImpl.c")
+endif()
+
 aux_source_directory(src/tq TQ_SRC)
 aux_source_directory(src/tsdb TSDB_SRC)
 aux_source_directory(src/vnd VND_SRC)
+
 list(APPEND
     VNODE_SRC
     ${META_SRC}
@@ -22,7 +39,6 @@ target_link_libraries(
     PUBLIC util
     PUBLIC common
     PUBLIC transport
-    PUBLIC bdb
     PUBLIC tfs
     PUBLIC wal
     PUBLIC scheduler
@@ -31,6 +47,12 @@ target_link_libraries(
     PUBLIC sync
 )
 
+if(${META_DB_IMPL} STREQUAL "BDB")
+    target_link_libraries(vnode PUBLIC bdb)
+elseif(${META_DB_IMPL} STREQUAL "TDB")
+    target_link_libraries(vnode PUBLIC tdb)
+endif()
+
 if(${BUILD_TEST})
     # add_subdirectory(test)
 endif(${BUILD_TEST})

source/dnode/vnode/src/inc/vnd.h

@@ -23,8 +23,8 @@
 #include "tlist.h"
 #include "tlockfree.h"
 #include "tmacro.h"
-#include "wal.h"
 #include "tq.h"
+#include "wal.h"
 
 #include "vnode.h"
 
@@ -175,7 +175,6 @@ void*           vmaMalloc(SVMemAllocator* pVMA, uint64_t size);
 void            vmaFree(SVMemAllocator* pVMA, void* ptr);
 bool            vmaIsFull(SVMemAllocator* pVMA);
 
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/meta/metaTDBImpl.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaDef.h"
+
+#include "tdb.h"
+
+struct SMetaDB {
+  TENV *pEnv;
+  TDB * pTbDB;
+  TDB * pSchemaDB;
+  TDB * pNameIdx;
+  TDB * pStbIdx;
+  TDB * pNtbIdx;
+  TDB * pCtbIdx;
+  // tag index hash table
+  // suid+colid --> TDB *
+  struct {
+  } tagIdxHt;
+};
+
+#define A(op, flag)                 \
+  do {                              \
+    if ((ret = op) != 0) goto flag; \
+  } while (0)
+
+int metaOpenDB(SMeta *pMeta) {
+  SMetaDB *pDb;
+  TENV *   pEnv;
+  TDB *    pTbDB;
+  TDB *    pSchemaDB;
+  TDB *    pNameIdx;
+  TDB *    pStbIdx;
+  TDB *    pNtbIdx;
+  TDB *    pCtbIdx;
+  int      ret;
+
+  pDb = (SMetaDB *)calloc(1, sizeof(*pDb));
+  if (pDb == NULL) {
+    return -1;
+  }
+
+  // Create and open the ENV
+  A((tdbEnvCreate(&pEnv)), _err);
+#if 0
+  // Set options of the environment
+  A(tdbEnvSetPageSize(pEnv, 8192), _err);
+  A(tdbEnvSetCacheSize(pEnv, 16 * 1024 * 1024), _err);
+#endif
+  A((tdbEnvOpen(&pEnv)), _err);
+
+  // Create and open each DB
+  A(tdbCreate(&pTbDB), _err);
+  A(tdbOpen(&pTbDB, "table.db", NULL, pEnv), _err);
+
+  A(tdbCreate(&pSchemaDB), _err);
+  A(tdbOpen(&pSchemaDB, "schema.db", NULL, pEnv), _err);
+
+  A(tdbCreate(&pNameIdx), _err);
+  A(tdbOpen(&pNameIdx, "name.db", NULL, pEnv), _err);
+  // tdbAssociate();
+
+  pDb->pEnv = pEnv;
+  pDb->pTbDB = pTbDB;
+  pDb->pSchemaDB = pSchemaDB;
+  pMeta->pDB = pDb;
+  return 0;
+
+_err:
+  return -1;
+}
+
+void metaCloseDB(SMeta *pMeta) {
+  // TODO
+}
+
+int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
+  // TODO
+  return 0;
+}
+
+int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
+  // TODO
+  return 0;
+}
+
+STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
+  // TODO
+  return NULL;
+}
+
+STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
+  // TODO
+  return NULL;
+}
+
+SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
+  // TODO
+  return NULL;
+}
+
+STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
+  // TODO
+  return NULL;
+}
+
+SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
+  // TODO
+  return NULL;
+}
+
+void metaCloseTbCursor(SMTbCursor *pTbCur) {
+  // TODO
+}
+
+char *metaTbCursorNext(SMTbCursor *pTbCur) {
+  // TODO
+  return NULL;
+}
+
+SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
+  // TODO
+  return NULL;
+}
+
+void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
+  // TODO
+}
+
+tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/inc/tdb.h

@@ -22,41 +22,44 @@
 extern "C" {
 #endif
 
-// #define TDB_EXTERN
-// #define TDB_PUBLIC
-// #define TDB_STATIC static
+typedef struct STDb       TDB;
+typedef struct STDbEnv    TENV;
+typedef struct STDbCurosr TDBC;
 
-// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t;
+typedef int32_t pgsz_t;
+typedef int32_t cachesz_t;
 
-// // Forward declarations
-// typedef struct TDB TDB;
-// // typedef struct TDB_MPOOL  TDB_MPOOL;
-// // typedef struct TDB_MPFILE TDB_MPFILE;
-// // typedef struct TDB_CURSOR TDB_CURSOR;
+typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
 
-// typedef struct {
-//   void*    bdata;
-//   uint32_t size;
-// } TDB_KEY, TDB_VALUE;
+// TEVN
+int tdbEnvCreate(TENV **ppEnv, const char *rootDir);
+int tdbEnvOpen(TENV *ppEnv);
+int tdbEnvClose(TENV *pEnv);
 
-// // TDB Operations
-// int tdbCreateDB(TDB** dbpp, tdb_db_t type);
-// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags);
-// int tdbCloseDB(TDB* dbp, uint32_t flags);
-// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags);
-// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags);
+int       tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize);
+pgsz_t    tdbEnvGetPageSize(TENV *pEnv);
+cachesz_t tdbEnvGetCacheSize(TENV *pEnv);
 
-// // TDB_MPOOL
-// int tdbOpenMPool(TDB_MPOOL** mp);
-// int tdbCloseMPool(TDB_MPOOL* mp);
+int tdbEnvBeginTxn(TENV *pEnv);
+int tdbEnvCommit(TENV *pEnv);
 
-// // TDB_MPFILE
-// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp);
-// int tdbCloseMPFile(TDB_MPFILE** mpf);
+// TDB
+int tdbCreate(TDB **ppDb);
+int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv);
+int tdbClose(TDB *pDb);
+int tdbDrop(TDB *pDb);
 
-// // TDB_CURSOR
-// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp);
-// int tdbCloseCurosr(TDB_CURSOR* tdbcp);
+int tdbSetKeyLen(TDB *pDb, int klen);
+int tdbSetValLen(TDB *pDb, int vlen);
+int tdbSetDup(TDB *pDb, int dup);
+int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn);
+int tdbGetKeyLen(TDB *pDb);
+int tdbGetValLen(TDB *pDb);
+int tdbGetDup(TDB *pDb);
+
+int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData);
+
+// TDBC
 
 #ifdef __cplusplus
 }

source/libs/tdb/src/db/tdb.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+struct STDb {
+  char         dbname[TDB_MAX_DBNAME_LEN];
+  SBTree *     pBt;      // current access method (may extend)
+  SPgFile *    pPgFile;  // backend page file this DB is using
+  TENV *       pEnv;     // TENV containing the DB
+  int          klen;     // key length if know
+  int          vlen;     // value length if know
+  bool         dup;      // dup mode
+  TdbKeyCmprFn cFn;      // compare function
+};
+
+struct STDbCurosr {
+  SBtCursor *pBtCur;
+};
+
+static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
+
+int tdbCreate(TDB **ppDb) {
+  TDB *pDb;
+
+  // create the handle
+  pDb = (TDB *)calloc(1, sizeof(*pDb));
+  if (pDb == NULL) {
+    return -1;
+  }
+
+  pDb->klen = TDB_VARIANT_LEN;
+  pDb->vlen = TDB_VARIANT_LEN;
+  pDb->dup = false;
+  pDb->cFn = tdbDefaultKeyCmprFn;
+
+  *ppDb = pDb;
+  return 0;
+}
+
+static int tdbDestroy(TDB *pDb) {
+  if (pDb) {
+    free(pDb);
+  }
+  return 0;
+}
+
+int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) {
+  int       ret;
+  uint8_t   fileid[TDB_FILE_ID_LEN];
+  SPgFile * pPgFile;
+  SPgCache *pPgCache;
+  SBTree *  pBt;
+  bool      fileExist;
+  size_t    dbNameLen;
+  pgno_t    dbRootPgno;
+  char      dbfname[128];  // TODO: make this as a macro or malloc on the heap
+
+  ASSERT(pDb != NULL);
+  ASSERT(fname != NULL);
+  // TODO: Here we simply put an assert here. In the future, make `pEnv`
+  // can be set as NULL.
+  ASSERT(pEnv != NULL);
+
+  // check the DB name
+  dbNameLen = 0;
+  if (dbname) {
+    dbNameLen = strlen(dbname);
+    if (dbNameLen >= TDB_MAX_DBNAME_LEN) {
+      return -1;
+    }
+
+    memcpy(pDb->dbname, dbname, dbNameLen);
+  }
+
+  pDb->dbname[dbNameLen] = '\0';
+
+  // get page file from the env, if not opened yet, open it
+  pPgFile = NULL;
+  snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname);
+  fileExist = (tdbCheckFileAccess(fname, TDB_F_OK) == 0);
+  if (fileExist) {
+    tdbGnrtFileID(dbfname, fileid, false);
+    pPgFile = tdbEnvGetPageFile(pEnv, fileid);
+  }
+
+  if (pPgFile == NULL) {
+    ret = pgFileOpen(&pPgFile, dbfname, pEnv);
+    if (ret != 0) {
+      // TODO: handle error
+      return -1;
+    }
+  }
+
+  // TODO: get the root page number from the master DB of the page file
+  // tdbGet(&dbRootPgno);
+  if (dbRootPgno == 0) {
+    // DB not exist, create one
+    ret = pgFileAllocatePage(pPgFile, &dbRootPgno);
+    if (ret != 0) {
+      // TODO: handle error
+    }
+    // tdbInsert(pPgFile->pMasterDB, dbname, strlen(dbname), &dbRootPgno, sizeof(dbRootPgno));
+  }
+
+  ASSERT(dbRootPgno > 1);
+
+  // pDb->pBt->root = dbRootPgno;
+
+  // register
+  pDb->pPgFile = pPgFile;
+  tdbEnvRgstDB(pEnv, pDb);
+  pDb->pEnv = pEnv;
+
+  return 0;
+}
+
+int tdbClose(TDB *pDb) {
+  if (pDb == NULL) return 0;
+  return tdbDestroy(pDb);
+}
+
+int tdbDrop(TDB *pDb) {
+  // TODO
+  return 0;
+}
+
+int tdbSetKeyLen(TDB *pDb, int klen) {
+  // TODO: check `klen`
+  pDb->klen = klen;
+  return 0;
+}
+
+int tdbSetValLen(TDB *pDb, int vlen) {
+  // TODO: check `vlen`
+  pDb->vlen = vlen;
+  return 0;
+}
+
+int tdbSetDup(TDB *pDb, int dup) {
+  if (dup) {
+    pDb->dup = true;
+  } else {
+    pDb->dup = false;
+  }
+  return 0;
+}
+
+int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn) {
+  if (fn == NULL) {
+    return -1;
+  } else {
+    pDb->cFn = fn;
+  }
+  return 0;
+}
+
+int tdbGetKeyLen(TDB *pDb) { return pDb->klen; }
+
+int tdbGetValLen(TDB *pDb) { return pDb->vlen; }
+
+int tdbGetDup(TDB *pDb) {
+  if (pDb->dup) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData) {
+  // TODO
+  return 0;
+}
+
+static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2) {
+  int mlen;
+  int cret;
+
+  ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL);
+
+  mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2;
+  cret = memcmp(pKey1, pKey2, mlen);
+  if (cret == 0) {
+    if (keyLen1 < keyLen2) {
+      cret = -1;
+    } else if (keyLen1 > keyLen2) {
+      cret = 1;
+    } else {
+      cret = 0;
+    }
+  }
+  return cret;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdbBtree.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+struct SBtCursor {
+  SBTree *pBtree;
+  pgno_t  pgno;
+  SPage * pPage;  // current page traversing
+};
+
+typedef struct {
+  pgno_t pgno;
+  pgsz_t offset;
+} SBtIdx;
+
+// Btree page header definition
+typedef struct __attribute__((__packed__)) {
+  uint8_t  flag;        // page flag
+  int32_t  vlen;        // value length of current page, TDB_VARIANT_LEN for variant length
+  uint16_t nPayloads;   // number of total payloads
+  pgoff_t  freeOff;     // free payload offset
+  pgsz_t   fragSize;    // total fragment size
+  pgoff_t  offPayload;  // payload offset
+  pgno_t   rChildPgno;  // right most child page number
+} SBtPgHdr;
+
+typedef int (*BtreeCmprFn)(const void *, const void *);
+
+#define BTREE_PAGE_HDR(pPage) NULL             /* TODO */
+#define BTREE_PAGE_PAYLOAD_AT(pPage, idx) NULL /*TODO*/
+#define BTREE_PAGE_IS_LEAF(pPage) 0            /* TODO */
+
+static int btreeCreate(SBTree **ppBt);
+static int btreeDestroy(SBTree *pBt);
+static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno);
+
+int btreeOpen(SBTree **ppBt, SPgFile *pPgFile) {
+  SBTree *pBt;
+  int     ret;
+
+  ret = btreeCreate(&pBt);
+  if (ret != 0) {
+    return -1;
+  }
+
+  *ppBt = pBt;
+  return 0;
+}
+
+int btreeClose(SBTree *pBt) {
+  // TODO
+  return 0;
+}
+
+static int btreeCreate(SBTree **ppBt) {
+  SBTree *pBt;
+
+  pBt = (SBTree *)calloc(1, sizeof(*pBt));
+  if (pBt == NULL) {
+    return -1;
+  }
+
+  // TODO
+  return 0;
+}
+
+static int btreeDestroy(SBTree *pBt) {
+  if (pBt) {
+    free(pBt);
+  }
+  return 0;
+}
+
+int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt) {
+  // TODO
+  return 0;
+}
+
+int btreeCursorClose(SBtCursor *pBtCur) {
+  // TODO
+  return 0;
+}
+
+int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey) {
+  SPage *     pPage;
+  SBtPgHdr *  pBtPgHdr;
+  SPgFile *   pPgFile;
+  pgno_t      childPgno;
+  pgno_t      rootPgno;
+  int         nPayloads;
+  void *      pPayload;
+  BtreeCmprFn cmpFn;
+
+  // 1. Move the cursor to the root page
+  if (rootPgno == TDB_IVLD_PGNO) {
+    // No any data in this btree, just return not found (TODO)
+    return 0;
+  } else {
+    // Load the page from the file by the SPgFile handle
+    pPage = pgFileFetch(pPgFile, rootPgno);
+
+    pBtCur->pPage = pPage;
+  }
+
+  // 2. Loop to search over the whole tree
+  for (;;) {
+    int lidx, ridx, midx, cret;
+
+    pPage = pBtCur->pPage;
+    pBtPgHdr = BTREE_PAGE_HDR(pPage);
+    nPayloads = pBtPgHdr->nPayloads;
+
+    // Binary search the page
+    lidx = 0;
+    ridx = nPayloads - 1;
+    midx = (lidx + ridx) >> 1;
+    for (;;) {
+      // get the payload ptr at midx
+      pPayload = BTREE_PAGE_PAYLOAD_AT(pPage, midx);
+
+      // the payload and the key
+      cret = cmpFn(pKey, pPayload);
+
+      if (cret < 0) {
+        /* TODO */
+      } else if (cret > 0) {
+        /* TODO */
+      } else {
+        /* TODO */
+      }
+
+      if (lidx > ridx) break;
+      midx = (lidx + ridx) >> 1;
+    }
+    if (BTREE_PAGE_IS_LEAF(pPage)) {
+      /* TODO */
+      break;
+    } else {
+      /* TODO */
+      btreeCursorMoveToChild(pBtCur, childPgno);
+    }
+  }
+
+  return 0;
+}
+
+static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno) {
+  SPgFile *pPgFile;
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdbEnv.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+struct STDbEnv {
+  char *      rootDir;    // root directory of the environment
+  char *      jname;      // journal file name
+  int         jfd;        // journal file fd
+  pgsz_t      pgSize;     // page size
+  cachesz_t   cacheSize;  // total cache size
+  STDbList    dbList;     // TDB List
+  SPgFileList pgfList;    // SPgFile List
+  SPgCache *  pPgCache;   // page cache
+  struct {
+#define TDB_ENV_PGF_HASH_BUCKETS 17
+    SPgFileList buckets[TDB_ENV_PGF_HASH_BUCKETS];
+  } pgfht;  // page file hash table;
+};
+
+#define TDB_ENV_PGF_HASH(fileid)        \
+  ({                                    \
+    uint8_t *tmp = (uint8_t *)(fileid); \
+    tmp[0] + tmp[1] + tmp[2];           \
+  })
+
+static int tdbEnvDestroy(TENV *pEnv);
+
+int tdbEnvCreate(TENV **ppEnv, const char *rootDir) {
+  TENV * pEnv;
+  size_t slen;
+  size_t jlen;
+
+  ASSERT(rootDir != NULL);
+
+  *ppEnv = NULL;
+  slen = strlen(rootDir);
+  jlen = slen + strlen(TDB_JOURNAL_NAME) + 1;
+  pEnv = (TENV *)calloc(1, sizeof(*pEnv) + slen + 1 + jlen + 1);
+  if (pEnv == NULL) {
+    return -1;
+  }
+
+  pEnv->rootDir = (char *)(&pEnv[1]);
+  pEnv->jname = pEnv->rootDir + slen + 1;
+  pEnv->jfd = -1;
+  pEnv->pgSize = TDB_DEFAULT_PGSIZE;
+  pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE;
+
+  memcpy(pEnv->rootDir, rootDir, slen);
+  pEnv->rootDir[slen] = '\0';
+  sprintf(pEnv->jname, "%s/%s", rootDir, TDB_JOURNAL_NAME);
+
+  TD_DLIST_INIT(&(pEnv->dbList));
+  TD_DLIST_INIT(&(pEnv->pgfList));
+
+  /* TODO */
+
+  *ppEnv = pEnv;
+  return 0;
+}
+
+int tdbEnvOpen(TENV *pEnv) {
+  SPgCache *pPgCache;
+  int       ret;
+
+  ASSERT(pEnv != NULL);
+
+  /* TODO: here we do not need to create the root directory, more
+   * work should be done here
+   */
+  mkdir(pEnv->rootDir, 0755);
+
+  ret = pgCacheOpen(&pPgCache, pEnv);
+  if (ret != 0) {
+    goto _err;
+  }
+
+  pEnv->pPgCache = pPgCache;
+  return 0;
+
+_err:
+  return -1;
+}
+
+int tdbEnvClose(TENV *pEnv) {
+  if (pEnv == NULL) return 0;
+  pgCacheClose(pEnv->pPgCache);
+  tdbEnvDestroy(pEnv);
+  return 0;
+}
+
+int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize) {
+  if (!TDB_IS_PGSIZE_VLD(pgSize) || cacheSize / pgSize < 10) {
+    return -1;
+  }
+
+  /* TODO */
+
+  pEnv->pgSize = pgSize;
+  pEnv->cacheSize = cacheSize;
+
+  return 0;
+}
+
+pgsz_t tdbEnvGetPageSize(TENV *pEnv) { return pEnv->pgSize; }
+
+cachesz_t tdbEnvGetCacheSize(TENV *pEnv) { return pEnv->cacheSize; }
+
+SPgFile *tdbEnvGetPageFile(TENV *pEnv, const uint8_t fileid[]) {
+  SPgFileList *pBucket;
+  SPgFile *    pPgFile;
+
+  pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(fileid) % TDB_ENV_PGF_HASH_BUCKETS);  // TODO
+  for (pPgFile = TD_DLIST_HEAD(pBucket); pPgFile != NULL; pPgFile = TD_DLIST_NODE_NEXT_WITH_FIELD(pPgFile, envHash)) {
+    if (memcmp(fileid, pPgFile->fileid, TDB_FILE_ID_LEN) == 0) break;
+  };
+
+  return pPgFile;
+}
+
+SPgCache *tdbEnvGetPgCache(TENV *pEnv) { return pEnv->pPgCache; }
+
+static int tdbEnvDestroy(TENV *pEnv) {
+  // TODO
+  return 0;
+}
+
+int tdbEnvBeginTxn(TENV *pEnv) {
+  pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755);
+  if (pEnv->jfd < 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int tdbEnvCommit(TENV *pEnv) {
+  /* TODO */
+  close(pEnv->jfd);
+  pEnv->jfd = -1;
+  return 0;
+}
+
+const char *tdbEnvGetRootDir(TENV *pEnv) { return pEnv->rootDir; }
+
+int tdbEnvRgstPageFile(TENV *pEnv, SPgFile *pPgFile) {
+  SPgFileList *pBucket;
+
+  TD_DLIST_APPEND_WITH_FIELD(&(pEnv->pgfList), pPgFile, envPgfList);
+
+  pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(pPgFile->fileid) % TDB_ENV_PGF_HASH_BUCKETS);  // TODO
+  TD_DLIST_APPEND_WITH_FIELD(pBucket, pPgFile, envHash);
+
+  return 0;
+}
+
+int tdbEnvRgstDB(TENV *pEnv, TDB *pDb) {
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdb_mpool.c → source/libs/tdb/src/db/tdbPgCache.c

@@ -12,10 +12,225 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#include "tdbInt.h"
+
+typedef TD_DLIST(SPage) SPgList;
+struct SPgCache {
+  TENV *  pEnv;  // TENV containing this page cache
+  pgsz_t  pgsize;
+  int32_t npage;
+  SPage **pages;
+  SPgList freeList;
+  SPgList lru;
+  struct {
+    int32_t  nbucket;
+    SPgList *buckets;
+  } pght;  // page hash table
+};
+
+static void pgCachePinPage(SPage *pPage);
+static void pgCacheUnpinPage(SPage *pPage);
+
+int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv) {
+  SPgCache *pPgCache;
+  SPage *   pPage;
+  void *    pData;
+  pgsz_t    pgSize;
+  cachesz_t cacheSize;
+  int32_t   npage;
+  int32_t   nbucket;
+  size_t    msize;
+
+  *ppPgCache = NULL;
+  pgSize = tdbEnvGetPageSize(pEnv);
+  cacheSize = tdbEnvGetCacheSize(pEnv);
+  npage = cacheSize / pgSize;
+  nbucket = npage;
+  msize = sizeof(*pPgCache) + sizeof(SPage *) * npage + sizeof(SPgList) * nbucket;
+
+  // Allocate the handle
+  pPgCache = (SPgCache *)calloc(1, msize);
+  if (pPgCache == NULL) {
+    return -1;
+  }
+
+  // Init the handle
+  pPgCache->pEnv = pEnv;
+  pPgCache->pgsize = pgSize;
+  pPgCache->npage = npage;
+  pPgCache->pages = (SPage **)(&pPgCache[1]);
+  pPgCache->pght.nbucket = nbucket;
+  pPgCache->pght.buckets = (SPgList *)(&(pPgCache->pages[npage]));
+
+  TD_DLIST_INIT(&(pPgCache->freeList));
+
+  for (int32_t i = 0; i < npage; i++) {
+    pData = malloc(pgSize + sizeof(SPage));
+    if (pData == NULL) {
+      return -1;
+      // TODO: handle error
+    }
+
+    pPage = POINTER_SHIFT(pData, pgSize);
+
+    pPage->pgid = TDB_IVLD_PGID;
+    pPage->frameid = i;
+    pPage->pData = pData;
+
+    // add current page to the page cache
+    pPgCache->pages[i] = pPage;
+    TD_DLIST_APPEND_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode);
+  }
+
+#if 0
+  for (int32_t i = 0; i < nbucket; i++) {
+    TD_DLIST_INIT(pPgCache->pght.buckets + i);
+  }
+#endif
+
+  *ppPgCache = pPgCache;
+  return 0;
+}
+
+int pgCacheClose(SPgCache *pPgCache) {
+  SPage *pPage;
+  if (pPgCache) {
+    for (int32_t i = 0; i < pPgCache->npage; i++) {
+      pPage = pPgCache->pages[i];
+      tfree(pPage->pData);
+    }
+
+    free(pPgCache);
+  }
+
+  return 0;
+}
+
+#define PG_CACHE_HASH(fileid, pgno)       \
+  ({                                      \
+    uint64_t *tmp = (uint64_t *)(fileid); \
+    (tmp[0] + tmp[1] + tmp[2] + (pgno));  \
+  })
+
+SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid) {
+  SPage *  pPage;
+  SPgFile *pPgFile;
+  SPgList *pBucket;
+
+  // 1. Search the page hash table SPgCache.pght
+  pBucket = pPgCache->pght.buckets + (PG_CACHE_HASH(pgid.fileid, pgid.pgno) % pPgCache->pght.nbucket);
+  pPage = TD_DLIST_HEAD(pBucket);
+  while (pPage && tdbCmprPgId(&(pPage->pgid), &pgid)) {
+    pPage = TD_DLIST_NODE_NEXT_WITH_FIELD(pPage, pghtNode);
+  }
 
-#include "tdb_mpool.h"
+  if (pPage) {
+    // Page is found, pin the page and return the page
+    pgCachePinPage(pPage);
+    return pPage;
+  }
+
+  // 2. Check the free list
+  pPage = TD_DLIST_HEAD(&(pPgCache->freeList));
+  if (pPage) {
+    TD_DLIST_POP_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode);
+    pgCachePinPage(pPage);
+    return pPage;
+  }
+
+  // 3. Try to recycle a page from the LRU list
+  pPage = TD_DLIST_HEAD(&(pPgCache->lru));
+  if (pPage) {
+    TD_DLIST_POP_WITH_FIELD(&(pPgCache->lru), pPage, lruNode);
+    // TODO: remove from the hash table
+    pgCachePinPage(pPage);
+    return pPage;
+  }
+
+  // 4. If a memory allocator is set, try to allocate from the allocator (TODO)
+
+  return NULL;
+}
+
+int pgCacheRelease(SPage *pPage) {
+  // TODO
+  return 0;
+}
+
+static void pgCachePinPage(SPage *pPage) {
+  // TODO
+}
+
+static void pgCacheUnpinPage(SPage *pPage) {
+  // TODO
+}
+
+#if 0
+// Exposed handle
+typedef struct TDB_MPOOL  TDB_MPOOL;
+typedef struct TDB_MPFILE TDB_MPFILE;
+
+typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t;
+typedef struct pg_t {
+  SRWLatch             rwLatch;
+  frame_id_t           frameid;
+  pgid_t               pgid;
+  uint8_t              dirty;
+  uint8_t              rbit;
+  int32_t              pinRef;
+  pg_free_dlist_node_t free;
+  pg_hash_dlist_node_t hash;
+  void *               p;
+} pg_t;
+
+typedef TD_DLIST(pg_t) pg_list_t;
+typedef struct {
+  SRWLatch latch;
+  TD_DLIST(TDB_MPFILE);
+} mpf_bucket_t;
+struct TDB_MPOOL {
+  int64_t    cachesize;
+  pgsz_t     pgsize;
+  int32_t    npages;
+  pg_t *     pages;
+  pg_list_t  freeList;
+  frame_id_t clockHand;
+  struct {
+    int32_t    nbucket;
+    pg_list_t *hashtab;
+  } pgtab;  // page table, hash<pgid_t, pg_t>
+  struct {
+#define MPF_HASH_BUCKETS 16
+    mpf_bucket_t buckets[MPF_HASH_BUCKETS];
+  } mpfht;  // MPF hash table. MPFs using this MP will be put in this hash table
+};
+
+#define MP_PAGE_AT(mp, idx) (mp)->pages[idx]
+
+typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t;
+struct TDB_MPFILE {
+  char *              fname;                    // file name
+  int                 fd;                       // fd
+  uint8_t             fileid[TDB_FILE_ID_LEN];  // file ID
+  TDB_MPOOL *         mp;                       // underlying memory pool
+  td_mpf_dlist_node_t node;
+};
+
+/*=================================================== Exposed apis ==================================================*/
+// TDB_MPOOL
+int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize);
+int tdbMPoolClose(TDB_MPOOL *mp);
+int tdbMPoolSync(TDB_MPOOL *mp);
+
+// TDB_MPFILE
+int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp);
+int tdbMPoolFileClose(TDB_MPFILE *mpf);
+int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr);
+int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr);
+int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
+int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
+int tdbMPoolFileSync(TDB_MPFILE *mpf);
 
-static int         tdbGnrtFileID(const char *fname, uint8_t *fileid);
 static void        tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
 static void        tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
 static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid);
@@ -23,7 +238,7 @@ static int         tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p);
 static int         tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p);
 static void        tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp);
 
-int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) {
+int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) {
   TDB_MPOOL *mp = NULL;
   size_t     tsize;
   pg_t *     pagep;
@@ -120,7 +335,7 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) {
     goto _err;
   }
 
-  if (tdbGnrtFileID(fname, mpf->fileid) < 0) {
+  if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) {
     goto _err;
   }
 
@@ -230,22 +445,6 @@ int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
   return 0;
 }
 
-static int tdbGnrtFileID(const char *fname, uint8_t *fileid) {
-  struct stat statbuf;
-
-  if (stat(fname, &statbuf) < 0) {
-    return -1;
-  }
-
-  memset(fileid, 0, TDB_FILE_ID_LEN);
-
-  ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino;
-  ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev;
-  ((uint64_t *)fileid)[2] = rand();
-
-  return 0;
-}
-
 #define MPF_GET_BUCKETID(fileid)                   \
   ({                                               \
     uint64_t *tmp = (uint64_t *)fileid;            \
@@ -317,7 +516,7 @@ static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
 }
 
 static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) {
-  pgsize_t   pgsize;
+  pgsz_t     pgsize;
   TDB_MPOOL *mp;
   off_t      offset;
   size_t     rsize;
@@ -334,7 +533,7 @@ static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) {
 }
 
 static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) {
-  pgsize_t   pgsize;
+  pgsz_t     pgsize;
   TDB_MPOOL *mp;
   off_t      offset;
 
@@ -376,4 +575,6 @@ static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) {
   } while (1);
 
   *pagepp = pagep;
-}
\ No newline at end of file
+}
+
+#endif
\ No newline at end of file

source/libs/tdb/src/db/tdbPgFile.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+typedef struct SPage1 {
+  char     magic[64];
+  pgno_t   mdbRootPgno;  // master DB root page number
+  pgno_t   freePgno;     // free list page number
+  uint32_t nFree;        // number of free pages
+} SPage1;
+
+typedef struct SFreePage {
+  /* TODO */
+} SFreePage;
+
+TDB_STATIC_ASSERT(sizeof(SPage1) <= TDB_MIN_PGSIZE, "TDB Page1 definition too large");
+
+static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData);
+
+int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
+  SPgFile * pPgFile;
+  SPgCache *pPgCache;
+  size_t    fnameLen;
+  pgno_t    fsize;
+
+  *ppPgFile = NULL;
+
+  // create the handle
+  fnameLen = strlen(fname);
+  pPgFile = (SPgFile *)calloc(1, sizeof(*pPgFile) + fnameLen + 1);
+  if (pPgFile == NULL) {
+    return -1;
+  }
+
+  ASSERT(pEnv != NULL);
+
+  // init the handle
+  pPgFile->fname = (char *)(&(pPgFile[1]));
+  memcpy(pPgFile->fname, fname, fnameLen);
+  pPgFile->fname[fnameLen] = '\0';
+  pPgFile->fd = -1;
+
+  pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755);
+  if (pPgFile->fd < 0) {
+    // TODO: handle error
+    return -1;
+  }
+
+  tdbGnrtFileID(fname, pPgFile->fileid, false);
+  tdbGetFileSize(fname, tdbEnvGetPageSize(pEnv), &fsize);
+
+  pPgFile->fsize = fsize;
+  pPgFile->lsize = fsize;
+
+  if (pPgFile->fsize == 0) {
+    // A created file
+    pgno_t pgno;
+    pgid_t pgid;
+
+    pgFileAllocatePage(pPgFile, &pgno);
+
+    ASSERT(pgno == 1);
+
+    memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN);
+    pgid.pgno = pgno;
+
+    pgCacheFetch(pPgCache, pgid);
+    // Need to allocate the first page as a description page
+  } else {
+    // An existing file
+  }
+
+  /* TODO: other open operations */
+
+  // add the page file to the environment
+  tdbEnvRgstPageFile(pEnv, pPgFile);
+  pPgFile->pEnv = pEnv;
+
+  *ppPgFile = pPgFile;
+  return 0;
+}
+
+int pgFileClose(SPgFile *pPgFile) {
+  if (pPgFile) {
+    if (pPgFile->fd >= 0) {
+      close(pPgFile->fd);
+    }
+
+    tfree(pPgFile->fname);
+    free(pPgFile);
+  }
+
+  return 0;
+}
+
+SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno) {
+  SPgCache *pPgCache;
+  SPage *   pPage;
+  pgid_t    pgid;
+
+  // 1. Fetch from the page cache
+  // pgCacheFetch(pPgCache, pgid);
+
+  // 2. If only get a page frame, no content, maybe
+  // need to load from the file
+  if (1 /*page not initialized*/) {
+    if (pgno < pPgFile->fsize) {
+      // load the page content from the disk
+      // ?? How about the freed pages ??
+    } else {
+      // zero the page, make the page as a empty
+      // page with zero records.
+    }
+  }
+
+#if 0
+  pPgCache = pPgFile->pPgCache;
+  pPage = NULL;
+  memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN);
+  pgid.pgno = pgno;
+
+  if (pgno > pPgFile->pgFileSize) {
+    // TODO
+  } else {
+    pPage = pgCacheFetch(pPgCache, pgid);
+    if (1 /*Page is cached, no need to load from file*/) {
+      return pPage;
+    } else {
+      // TODO: handle error
+      if (pgFileRead(pPgFile, pgno, (void *)pPage) < 0) {
+        // todoerr
+      }
+      return pPage;
+    }
+  }
+#endif
+
+  return pPage;
+}
+
+int pgFileRelease(SPage *pPage) {
+  pgCacheRelease(pPage);
+  return 0;
+}
+
+int pgFileWrite(SPage *pPage) {
+  // TODO
+  return 0;
+}
+
+int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno) {
+  pgno_t    pgno;
+  SPage1 *  pPage1;
+  SPgCache *pPgCache;
+  pgid_t    pgid;
+  SPage *   pPage;
+
+  if (pPgFile->lsize == 0) {
+    pgno = ++(pPgFile->lsize);
+  } else {
+    if (0) {
+      // TODO: allocate from the free list
+      pPage = pgCacheFetch(pPgCache, pgid);
+
+      if (pPage1->nFree > 0) {
+        // TODO
+      } else {
+        pgno = ++(pPgFile->lsize);
+      }
+    } else {
+      pgno = ++(pPgFile->lsize);
+    }
+  }
+
+  *pPgno = pgno;
+  return 0;
+}
+
+static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) {
+  pgsz_t   pgSize;
+  ssize_t  rsize;
+  uint8_t *pTData;
+  size_t   szToRead;
+
+#if 0
+
+  // pgSize = ; (TODO)
+  pTData = pData;
+  szToRead = pgSize;
+  for (; szToRead > 0;) {
+    rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize);
+    if (rsize < 0) {
+      if (errno == EINTR) {
+        continue;
+      } else {
+        return -1;
+      }
+    } else if (rsize == 0) {
+      return -1;
+    }
+
+    szToRead -= rsize;
+    pTData += rsize;
+  }
+#endif
+
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdbUtil.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
+  struct stat statbuf;
+
+  if (stat(fname, &statbuf) < 0) {
+    return -1;
+  }
+
+  memset(fileid, 0, TDB_FILE_ID_LEN);
+
+  ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino;
+  ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev;
+  if (unique) {
+    ((uint64_t *)fileid)[2] = rand();
+  }
+
+  return 0;
+}
+
+int tdbCheckFileAccess(const char *pathname, int mode) {
+  int flags = 0;
+
+  if (mode & TDB_F_OK) {
+    flags |= F_OK;
+  }
+
+  if (mode & TDB_R_OK) {
+    flags |= R_OK;
+  }
+
+  if (mode & TDB_W_OK) {
+    flags |= W_OK;
+  }
+
+  return access(pathname, flags);
+}
+
+int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) {
+  struct stat st;
+  int         ret;
+
+  ret = stat(fname, &st);
+  if (ret != 0) {
+    return -1;
+  }
+
+  ASSERT(st.st_size % pgSize == 0);
+
+  *pSize = st.st_size / pgSize;
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/inc/tdb_db.h → source/libs/tdb/src/inc/tdbBtree.h

@@ -13,33 +13,32 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TDB_DB_H_
-#define _TD_TDB_DB_H_
-
-#include "tdb_mpool.h"
+#ifndef _TD_BTREE_H_
+#define _TD_BTREE_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct TDB TDB;
-
-struct TDB {
-  char *      fname;
-  char *      dbname;
-  TDB_MPFILE *mpf;
-  // union {
-  //   TDB_BTREE *btree;
-  //   TDB_HASH * hash;
-  //   TDB_HEAP * heap;
-  // } dbam;  // db access method
-};
+typedef struct SBTree    SBTree;
+typedef struct SBtCursor SBtCursor;
+
+// SBTree
+int btreeOpen(SBTree **ppBt, SPgFile *pPgFile);
+int btreeClose(SBTree *pBt);
 
-int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags);
-int tdbClose(TDB *dbp, uint32_t flags);
+// SBtCursor
+int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt);
+int btreeCursorClose(SBtCursor *pBtCur);
+int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey);
+int btreeCursorNext(SBtCursor *pBtCur);
+
+struct SBTree {
+  pgno_t   root;
+};
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TDB_DB_H_*/
\ No newline at end of file
+#endif /*_TD_BTREE_H_*/
\ No newline at end of file

source/libs/tdb/src/db/tdb_db.c → source/libs/tdb/src/inc/tdbEnv.h

@@ -13,14 +13,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tdb_db.h"
+#ifndef _TDB_ENV_H_
+#define _TDB_ENV_H_
 
-int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) {
-  // TODO
-  return 0;
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+const char* tdbEnvGetRootDir(TENV* pEnv);
+SPgFile*    tdbEnvGetPageFile(TENV* pEnv, const uint8_t fileid[]);
+SPgCache*   tdbEnvGetPgCache(TENV* pEnv);
+int         tdbEnvRgstPageFile(TENV* pEnv, SPgFile* pPgFile);
+int         tdbEnvRgstDB(TENV* pEnv, TDB* pDb);
+
+#ifdef __cplusplus
 }
+#endif
 
-int tdbClose(TDB *dbp, uint32_t flags) {
-  // TODO
-  return 0;
-}
\ No newline at end of file
+#endif /*_TDB_ENV_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TDB_INTERNAL_H_
+#define _TD_TDB_INTERNAL_H_
+
+#include "tlist.h"
+#include "tlockfree.h"
+
+#include "tdb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SPgFile SPgFile;
+
+// pgno_t
+typedef int32_t pgno_t;
+#define TDB_IVLD_PGNO ((pgno_t)0)
+
+// fileid
+#define TDB_FILE_ID_LEN 24
+
+// pgid_t
+typedef struct {
+  uint8_t fileid[TDB_FILE_ID_LEN];
+  pgno_t  pgno;
+} pgid_t;
+
+#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO};
+
+static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
+  pgid_t *pgid1 = (pgid_t *)p1;
+  pgid_t *pgid2 = (pgid_t *)p2;
+  int     rcode;
+
+  rcode = memcmp(pgid1->fileid, pgid2->fileid, TDB_FILE_ID_LEN);
+  if (rcode) {
+    return rcode;
+  } else {
+    if (pgid1->pgno > pgid2->pgno) {
+      return 1;
+    } else if (pgid1->pgno < pgid2->pgno) {
+      return -1;
+    } else {
+      return 0;
+    }
+  }
+}
+
+// framd_id_t
+typedef int32_t frame_id_t;
+
+// pgsz_t
+#define TDB_MIN_PGSIZE 512
+#define TDB_MAX_PGSIZE 65536
+#define TDB_DEFAULT_PGSIZE 4096
+#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
+
+// pgoff_t
+typedef pgsz_t pgoff_t;
+
+// cache
+#define TDB_DEFAULT_CACHE_SIZE (256 * 4096)  // 1M
+
+// dbname
+#define TDB_MAX_DBNAME_LEN 24
+
+// tdb_log
+#define tdbError(var)
+
+typedef TD_DLIST(STDb) STDbList;
+typedef TD_DLIST(SPgFile) SPgFileList;
+typedef TD_DLIST_NODE(SPgFile) SPgFileListNode;
+
+#define TERR_A(val, op, flag)  \
+  do {                         \
+    if (((val) = (op)) != 0) { \
+      goto flag;               \
+    }                          \
+  } while (0)
+
+#define TERR_B(val, op, flag)     \
+  do {                            \
+    if (((val) = (op)) == NULL) { \
+      goto flag;                  \
+    }                             \
+  } while (0)
+
+#define TDB_VARIANT_LEN (int)-1
+
+// page payload format
+// <keyLen> + <valLen> + [key] + [value]
+#define TDB_DECODE_PAYLOAD(pPayload, keyLen, pKey, valLen, pVal) \
+  do {                                                           \
+    if ((keyLen) == TDB_VARIANT_LEN) {                           \
+      /* TODO: decode the keyLen */                              \
+    }                                                            \
+    if ((valLen) == TDB_VARIANT_LEN) {                           \
+      /* TODO: decode the valLen */                              \
+    }                                                            \
+    /* TODO */                                                   \
+  } while (0)
+
+#define TDB_JOURNAL_NAME "tdb.journal"
+
+#include "tdbUtil.h"
+
+#include "tdbBtree.h"
+
+#include "tdbPgCache.h"
+
+#include "tdbPgFile.h"
+
+#include "tdbEnv.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TDB_INTERNAL_H_*/

source/libs/tdb/src/inc/tdbPgCache.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_PAGE_CACHE_H_
+#define _TD_PAGE_CACHE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SPgCache SPgCache;
+typedef struct SPage    SPage;
+
+// SPgCache
+int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv);
+int pgCacheClose(SPgCache *pPgCache);
+
+SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid);
+int    pgCacheRelease(SPage *pPage);
+
+// SPage
+typedef TD_DLIST_NODE(SPage) SPgListNode;
+struct SPage {
+  pgid_t      pgid;      // page id
+  frame_id_t  frameid;   // frame id
+  uint8_t *   pData;     // real data
+  SPgListNode freeNode;  // for SPgCache.freeList
+  SPgListNode pghtNode;  // for pght
+  SPgListNode lruNode;   // for LRU
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_PAGE_CACHE_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbPgFile.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_PAGE_FILE_H_
+#define _TD_PAGE_FILE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct __attribute__((__packed__)) {
+  char    hdrInfo[16];  // info string
+  pgsz_t  szPage;       // page size of current file
+  int32_t cno;          // commit number counter
+  pgno_t  freePgno;     // freelist page number
+  uint8_t resv[100];    // reserved space
+} SPgFileHdr;
+
+#define TDB_PG_FILE_HDR_SIZE 128
+
+TDB_STATIC_ASSERT(sizeof(SPgFileHdr) == TDB_PG_FILE_HDR_SIZE, "Page file header size if not 128");
+
+struct SPgFile {
+  TENV *          pEnv;                     // env containing this page file
+  char *          fname;                    // backend file name
+  uint8_t         fileid[TDB_FILE_ID_LEN];  // file id
+  pgno_t          lsize;                    // page file logical size (for count)
+  pgno_t          fsize;                    // real file size on disk (for rollback)
+  int             fd;
+  SPgFileListNode envHash;
+  SPgFileListNode envPgfList;
+};
+
+int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv);
+int pgFileClose(SPgFile *pPgFile);
+
+SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno);
+int    pgFileRelease(SPage *pPage);
+
+int pgFileWrite(SPage *pPage);
+int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_PAGE_FILE_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdb_inc.h → source/libs/tdb/src/inc/tdbUtil.h

@@ -13,46 +13,32 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TDB_INC_H_
-#define _TD_TDB_INC_H_
-
-#include "os.h"
-#include "tlist.h"
-#include "tlockfree.h"
+#ifndef _TDB_UTIL_H_
+#define _TDB_UTIL_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-// pgno_t
-typedef int32_t pgno_t;
-#define TDB_IVLD_PGNO ((pgno_t)-1)
-
-// fileid
-#define TDB_FILE_ID_LEN 24
+#if __STDC_VERSION__ >= 201112L
+#define TDB_STATIC_ASSERT(op, info) static_assert(op, info)
+#else
+#define TDB_STATIC_ASSERT(op, info)
+#endif
 
-// pgid_t
-typedef struct {
-  uint8_t fileid[TDB_FILE_ID_LEN];
-  pgno_t  pgno;
-} pgid_t;
-#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO};
+#define TDB_ROUND8(x) (((x) + 7) & ~7)
 
-// framd_id_t
-typedef int32_t frame_id_t;
+int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique);
 
-// pgsize_t
-typedef int32_t pgsize_t;
-#define TDB_MIN_PGSIZE 512
-#define TDB_MAX_PGSIZE 16384
-#define TDB_DEFAULT_PGSIZE 4096
-#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
+#define TDB_F_OK 0x1
+#define TDB_R_OK 0x2
+#define TDB_W_OK 0x4
+int tdbCheckFileAccess(const char *pathname, int mode);
 
-// tdb_log
-#define tdbError(var)
+int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TDB_INC_H_*/
+#endif /*_TDB_UTIL_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdb_mpool.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TDB_MPOOL_H_
-#define _TD_TDB_MPOOL_H_
-
-#include "tdb_inc.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// Exposed handle
-typedef struct TDB_MPOOL  TDB_MPOOL;
-typedef struct TDB_MPFILE TDB_MPFILE;
-
-typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t;
-typedef struct pg_t {
-  SRWLatch             rwLatch;
-  frame_id_t           frameid;
-  pgid_t               pgid;
-  uint8_t              dirty;
-  uint8_t              rbit;
-  int32_t              pinRef;
-  pg_free_dlist_node_t free;
-  pg_hash_dlist_node_t hash;
-  void *               p;
-} pg_t;
-
-typedef TD_DLIST(pg_t) pg_list_t;
-typedef struct {
-  SRWLatch latch;
-  TD_DLIST(TDB_MPFILE);
-} mpf_bucket_t;
-struct TDB_MPOOL {
-  int64_t    cachesize;
-  pgsize_t   pgsize;
-  int32_t    npages;
-  pg_t *     pages;
-  pg_list_t  freeList;
-  frame_id_t clockHand;
-  struct {
-    int32_t    nbucket;
-    pg_list_t *hashtab;
-  } pgtab;  // page table, hash<pgid_t, pg_t>
-  struct {
-#define MPF_HASH_BUCKETS 16
-    mpf_bucket_t buckets[MPF_HASH_BUCKETS];
-  } mpfht;  // MPF hash table. MPFs using this MP will be put in this hash table
-};
-
-#define MP_PAGE_AT(mp, idx) (mp)->pages[idx]
-
-typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t;
-struct TDB_MPFILE {
-  char *              fname;                    // file name
-  int                 fd;                       // fd
-  uint8_t             fileid[TDB_FILE_ID_LEN];  // file ID
-  TDB_MPOOL *         mp;                       // underlying memory pool
-  td_mpf_dlist_node_t node;
-};
-
-/*=================================================== Exposed apis ==================================================*/
-// TDB_MPOOL
-int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize);
-int tdbMPoolClose(TDB_MPOOL *mp);
-int tdbMPoolSync(TDB_MPOOL *mp);
-
-// TDB_MPFILE
-int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp);
-int tdbMPoolFileClose(TDB_MPFILE *mpf);
-int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr);
-int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr);
-int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
-int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
-int tdbMPoolFileSync(TDB_MPFILE *mpf);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TDB_MPOOL_H_*/
\ No newline at end of file

source/libs/tdb/test/CMakeLists.txt

-# tdbMPoolTest
-add_executable(tdbMPoolTest "tdbMPoolTest.cpp")
-target_link_libraries(tdbMPoolTest tdb gtest gtest_main)
-
 # tdbTest
 add_executable(tdbTest "tdbTest.cpp")
 target_link_libraries(tdbTest tdb gtest gtest_main)
\ No newline at end of file

source/libs/tdb/test/tdbMPoolTest.cpp

-#include "gtest/gtest.h"
-
-#include <iostream>
-
-#include "tdb_mpool.h"
-
-TEST(tdb_mpool_test, test1) {
-  TDB_MPOOL * mp;
-  TDB_MPFILE *mpf;
-  pgno_t      pgno;
-  void *      pgdata;
-
-  // open mp
-  tdbMPoolOpen(&mp, 16384, 4096);
-
-  // open mpf
-  tdbMPoolFileOpen(&mpf, "test.db", mp);
-
-#define TEST1_TOTAL_PAGES 100
-  for (int i = 0; i < TEST1_TOTAL_PAGES; i++) {
-    tdbMPoolFileNewPage(mpf, &pgno, pgdata);
-
-    *(pgno_t *)pgdata = i;
-  }
-
-  // close mpf
-  tdbMPoolFileClose(mpf);
-
-  // close mp
-  tdbMPoolClose(mp);
-}

source/libs/tdb/test/tdbTest.cpp

@@ -2,13 +2,67 @@
 
 #include "tdb.h"
 
-TEST(tdb_api_test, tdb_create_open_close_db_test) {
-  // int  ret;
-  // TDB *dbp;
+TEST(tdb_test, simple_test) {
+  TENV *    pEnv;
+  TDB *     pDb1, *pDb2, *pDb3;
+  pgsz_t    pgSize = 1024;
+  cachesz_t cacheSize = 10240;
 
-  // tdbCreateDB(&dbp, TDB_BTREE_T);
+  // ENV
+  GTEST_ASSERT_EQ(tdbEnvCreate(&pEnv, "./testtdb"), 0);
 
-  // tdbOpenDB(dbp, 0);
+  GTEST_ASSERT_EQ(tdbEnvSetCache(pEnv, pgSize, cacheSize), 0);
 
-  // tdbCloseDB(dbp, 0);
+  GTEST_ASSERT_EQ(tdbEnvGetCacheSize(pEnv), cacheSize);
+
+  GTEST_ASSERT_EQ(tdbEnvGetPageSize(pEnv), pgSize);
+
+  GTEST_ASSERT_EQ(tdbEnvOpen(pEnv), 0);
+
+#if 1
+  // DB
+  GTEST_ASSERT_EQ(tdbCreate(&pDb1), 0);
+
+  // GTEST_ASSERT_EQ(tdbSetKeyLen(pDb1, 8), 0);
+
+  // GTEST_ASSERT_EQ(tdbGetKeyLen(pDb1), 8);
+
+  // GTEST_ASSERT_EQ(tdbSetValLen(pDb1, 3), 0);
+
+  // GTEST_ASSERT_EQ(tdbGetValLen(pDb1), 3);
+
+  // GTEST_ASSERT_EQ(tdbSetDup(pDb1, 1), 0);
+
+  // GTEST_ASSERT_EQ(tdbGetDup(pDb1), 1);
+
+  // GTEST_ASSERT_EQ(tdbSetCmprFunc(pDb1, NULL), 0);
+
+  tdbEnvBeginTxn(pEnv);
+
+  GTEST_ASSERT_EQ(tdbOpen(pDb1, "db.db", "db1", pEnv), 0);
+
+  // char *key = "key1";
+  // char *val = "value1";
+  // tdbInsert(pDb1, (void *)key, strlen(key), (void *)val, strlen(val));
+
+  tdbEnvCommit(pEnv);
+
+#if 0
+  // Insert
+
+  // Query
+
+  // Delete
+
+  // Query
+#endif
+
+  // GTEST_ASSERT_EQ(tdbOpen(&pDb2, "db.db", "db2", pEnv), 0);
+  // GTEST_ASSERT_EQ(tdbOpen(&pDb3, "index.db", NULL, pEnv), 0);
+  // tdbClose(pDb3);
+  // tdbClose(pDb2);
+  tdbClose(pDb1);
+#endif
+
+  tdbEnvClose(pEnv);
 }
\ No newline at end of file