Merge pull request #9992 from taosdata/feature/tkv

Feature/tkv

include/util/tlist.h

@@ -34,6 +34,7 @@ extern "C" {
 #define TD_SLIST_HEAD(sl) ((sl)->sl_head_)
 #define TD_SLIST_NELES(sl) ((sl)->sl_neles_)
 #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_)
+#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) ((sln)->field.sl_next_)
 
 #define TD_SLIST_INIT(sl)  \
   do {                     \
@@ -48,12 +49,25 @@ extern "C" {
     TD_SLIST_NELES(sl) += 1;                     \
   } while (0)
 
+#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, field)                   \
+  do {                                                             \
+    TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) = TD_SLIST_HEAD(sl); \
+    TD_SLIST_HEAD(sl) = (sln);                                     \
+    TD_SLIST_NELES(sl) += 1;                                       \
+  } while (0)
+
 #define TD_SLIST_POP(sl)                                       \
   do {                                                         \
     TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \
     TD_SLIST_NELES(sl) -= 1;                                   \
   } while (0)
 
+#define TD_SLIST_POP_WITH_FIELD(sl, field)                                       \
+  do {                                                                           \
+    TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), field); \
+    TD_SLIST_NELES(sl) -= 1;                                                     \
+  } while (0)
+
 // Double linked list ================
 #define TD_DLIST_NODE(TYPE) \
   struct {                  \
@@ -70,6 +84,8 @@ extern "C" {
 
 #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_)
 #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_)
+#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) ((dln)->field.dl_prev_)
+#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) ((dln)->field.dl_next_)
 #define TD_DLIST_HEAD(dl) ((dl)->dl_head_)
 #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_)
 #define TD_DLIST_NELES(dl) ((dl)->dl_neles_)
@@ -94,6 +110,20 @@ extern "C" {
     TD_DLIST_NELES(dl) += 1;                                    \
   } while (0)
 
+#define TD_DLIST_APPEND_WITH_FIELD(dl, dln, field)                                                  \
+  do {                                                                                              \
+    if (TD_DLIST_HEAD(dl) == NULL) {                                                                \
+      TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \
+      TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln);                                                \
+    } else {                                                                                        \
+      TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_TAIL(dl);                                \
+      TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL;                                             \
+      TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), field) = (dln);                              \
+      TD_DLIST_TAIL(dl) = (dln);                                                                    \
+    }                                                                                               \
+    TD_DLIST_NELES(dl) += 1;                                                                        \
+  } while (0)
+
 #define TD_DLIST_PREPEND(dl, dln)                               \
   do {                                                          \
     if (TD_DLIST_HEAD(dl) == NULL) {                            \
@@ -108,6 +138,20 @@ extern "C" {
     TD_DLIST_NELES(dl) += 1;                                    \
   } while (0)
 
+#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, field)                                                 \
+  do {                                                                                              \
+    if (TD_DLIST_HEAD(dl) == NULL) {                                                                \
+      TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \
+      TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln);                                                \
+    } else {                                                                                        \
+      TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = NULL;                                             \
+      TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = TD_DLIST_HEAD(dl);                                \
+      TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), field) = (dln);                              \
+      TD_DLIST_HEAD(dl) = (dln);                                                                    \
+    }                                                                                               \
+    TD_DLIST_NELES(dl) += 1;                                                                        \
+  } while (0)
+
 #define TD_DLIST_POP(dl, dln)                                                \
   do {                                                                       \
     if (TD_DLIST_HEAD(dl) == (dln)) {                                        \
@@ -126,6 +170,26 @@ extern "C" {
     TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL;                \
   } while (0)
 
+#define TD_DLIST_POP_WITH_FIELD(dl, dln, field)                                                   \
+  do {                                                                                            \
+    if (TD_DLIST_HEAD(dl) == (dln)) {                                                             \
+      TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field);                              \
+    }                                                                                             \
+    if (TD_DLIST_TAIL(dl) == (dln)) {                                                             \
+      TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, field);                              \
+    }                                                                                             \
+    if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) != NULL) {                                      \
+      TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, field), field) =           \
+          TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field);                                              \
+    }                                                                                             \
+    if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) != NULL) {                                      \
+      TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field), field) =           \
+          TD_DLIST_NODE_PREV_WITH_FIELD(dln, field);                                              \
+    }                                                                                             \
+    TD_DLIST_NELES(dl) -= 1;                                                                      \
+    TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \
+  } while (0)
+
 // General double linked list
 typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T;
 

source/dnode/vnode/src/meta/metaBDBImpl.c

@@ -587,7 +587,11 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
       pBuf = value.data;
       metaDecodeTbInfo(pBuf, &tbCfg);
       if (tbCfg.type == META_SUPER_TABLE) {
+        free(tbCfg.name);
+        free(tbCfg.stbCfg.pTagSchema);
         continue;
+      } else if (tbCfg.type == META_CHILD_TABLE) {
+        kvRowFree(tbCfg.ctbCfg.pTag);
       }
       return tbCfg.name;
     } else {

source/dnode/vnode/src/tq/tq.c

@@ -81,6 +81,10 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
 }
 
 void tqClose(STQ* pTq) {
+  if (pTq) {
+    tfree(pTq->path);
+    free(pTq);
+  }
   // TODO
 }
 

source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c

@@ -114,4 +114,4 @@ static void vArenaNodeFree(SVArenaNode *pNode) {
   if (pNode) {
     free(pNode);
   }
-}
\ No newline at end of file
+}

source/dnode/vnode/src/vnd/vnodeBufferPool.c

@@ -71,6 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) {
 
 void vnodeCloseBufPool(SVnode *pVnode) {
   if (pVnode->pBufPool) {
+    tfree(pVnode->pBufPool->pMAF);
     vmaDestroy(pVnode->pBufPool->inuse);
 
     while (true) {

source/dnode/vnode/src/vnd/vnodeMgr.c

@@ -119,6 +119,7 @@ static void* loop(void* arg) {
     pthread_mutex_unlock(&(vnodeMgr.mutex));
 
     (*(pTask->execute))(pTask->arg);
+    free(pTask);
   }
 
   return NULL;

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -145,6 +145,15 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 
 _exit:
 
+  free(pSW->pSchema);
+  free(pSW);
+  free(pTbCfg->name);
+  free(pTbCfg);
+  if (pTbCfg->type == META_SUPER_TABLE) {
+    free(pTbCfg->stbCfg.pTagSchema);
+  } else if (pTbCfg->type == META_SUPER_TABLE) {
+    kvRowFree(pTbCfg->ctbCfg.pTag);
+  }
   rpcMsg.handle = pMsg->handle;
   rpcMsg.ahandle = pMsg->ahandle;
   rpcMsg.pCont = pTbMetaMsg;
@@ -156,8 +165,8 @@ _exit:
   return 0;
 }
 
-static void freeItemHelper(void* pItem) {
-  char* p = *(char**)pItem;
+static void freeItemHelper(void *pItem) {
+  char *p = *(char **)pItem;
   free(p);
 }
 
@@ -187,14 +196,14 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
   // TODO: temp debug, and should del when show tables command ok
   vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables);
   if (numOfTables > 10000) {
-     numOfTables = 10000;
+    numOfTables = 10000;
   }
 
   metaCloseTbCursor(pCur);
 
   int32_t rowLen =
       (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4;
-  //int32_t numOfTables = (int32_t)taosArrayGetSize(pArray);
+  // int32_t numOfTables = (int32_t)taosArrayGetSize(pArray);
 
   int32_t payloadLen = rowLen * numOfTables;
   //  SVShowTablesFetchReq *pFetchReq = pMsg->pCont;
@@ -208,6 +217,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
     STR_TO_VARSTR(p, n);
 
     p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE);
+    free(n);
   }
 
   pFetchRsp->numOfRows = htonl(numOfTables);
@@ -222,6 +232,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
   };
 
   rpcSendResponse(&rpcMsg);
+
   taosArrayDestroyEx(pArray, freeItemHelper);
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
     pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
 
     // ser request version
-    void   *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+    void *  pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
     int64_t ver = pVnode->state.processed++;
     taosEncodeFixedU64(&pBuf, ver);
 
@@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
 int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
   SVCreateTbReq      vCreateTbReq;
   SVCreateTbBatchReq vCreateTbBatchReq;
-  void              *ptr = vnodeMalloc(pVnode, pMsg->contLen);
+  void *             ptr = vnodeMalloc(pVnode, pMsg->contLen);
   if (ptr == NULL) {
     // TODO: handle error
   }
@@ -76,6 +76,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       }
 
       // TODO: maybe need to clear the requst struct
+      free(vCreateTbReq.stbCfg.pSchema);
+      free(vCreateTbReq.stbCfg.pTagSchema);
+      free(vCreateTbReq.name);
       break;
     case TDMT_VND_CREATE_TABLE:
       tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq);
@@ -129,7 +132,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       // TODO: handle error
     }
   }
-  
+
   return 0;
 }
 

source/libs/tdb/CMakeLists.txt

 
-set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr")
+set(TDB_SUBDIRS "db")
 foreach(TDB_SUBDIR ${TDB_SUBDIRS})
     aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC)
 endforeach()

source/libs/tdb/inc/tdb.h

@@ -22,30 +22,41 @@
 extern "C" {
 #endif
 
-#define TDB_EXTERN
-#define TDB_PUBLIC
-#define TDB_STATIC static
-
-typedef enum {
-  TDB_BTREE_T = 0,
-  TDB_HASH_T,
-  TDB_HEAP_T,
-} tdb_db_t;
-
-// Forward declaration
-typedef struct TDB        TDB;
-typedef struct TDB_CURSOR TDB_CURSOR;
-
-// SKey
-typedef struct {
-  void*    bdata;
-  uint32_t size;
-} TDB_KEY, TDB_VALUE;
-
-// TDB Operations
-TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type);
-TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags);
-TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags);
+// #define TDB_EXTERN
+// #define TDB_PUBLIC
+// #define TDB_STATIC static
+
+// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t;
+
+// // Forward declarations
+// typedef struct TDB TDB;
+// // typedef struct TDB_MPOOL  TDB_MPOOL;
+// // typedef struct TDB_MPFILE TDB_MPFILE;
+// // typedef struct TDB_CURSOR TDB_CURSOR;
+
+// typedef struct {
+//   void*    bdata;
+//   uint32_t size;
+// } TDB_KEY, TDB_VALUE;
+
+// // TDB Operations
+// int tdbCreateDB(TDB** dbpp, tdb_db_t type);
+// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags);
+// int tdbCloseDB(TDB* dbp, uint32_t flags);
+// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags);
+// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags);
+
+// // TDB_MPOOL
+// int tdbOpenMPool(TDB_MPOOL** mp);
+// int tdbCloseMPool(TDB_MPOOL* mp);
+
+// // TDB_MPFILE
+// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp);
+// int tdbCloseMPFile(TDB_MPFILE** mpf);
+
+// // TDB_CURSOR
+// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp);
+// int tdbCloseCurosr(TDB_CURSOR* tdbcp);
 
 #ifdef __cplusplus
 }

source/libs/tdb/src/db/tdbDB.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "tdbDB.h"
-#include "tdb.h"
-
-TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) {
-  TDB* dbp;
-  int  ret;
-
-  dbp = calloc(1, sizeof(*dbp));
-  if (dbp == NULL) {
-    return -1;
-  }
-
-  dbp->pageSize = TDB_DEFAULT_PGSIZE;
-  dbp->type = type;
-
-  switch (type) {
-    case TDB_BTREE_T:
-      //   ret = tdbInitBtreeDB(dbp);
-      //   if (ret < 0) goto _err;
-      break;
-    case TDB_HASH_T:
-      //   ret = tdbInitHashDB(dbp);
-      //   if (ret < 0) goto _err;
-      break;
-    case TDB_HEAP_T:
-      //   ret = tdbInitHeapDB(dbp);
-      //   if (ret < 0) goto _err;
-      break;
-    default:
-      break;
-  }
-
-  *dbpp = dbp;
-  return 0;
-
-_err:
-  if (dbp) {
-    free(dbp);
-  }
-  *dbpp = NULL;
-  return 0;
-}
-
-TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags) {
-  int ret = 0;
-
-  if ((dbp->fname = strdup(fname)) == NULL) {
-    ret = -1;
-    return ret;
-  }
-
-  // Create the backup file if the file not exists
-
-  // Open the file as a sub-db or a master-db
-  if (dbname) {
-    if ((dbp->dbname = strdup(dbname)) == NULL) {
-      ret = -1;
-      return ret;
-    }
-    // TODO: Open the DB as a SUB-DB in this file
-  } else {
-    // TODO: Open the DB as a MASTER-DB in this file
-  }
-
-  return ret;
-}
-
-TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) {
-  // TODO
-  return 0;
-}
\ No newline at end of file

source/libs/tdb/src/inc/tdbHeap.h → source/libs/tdb/src/db/tdb_db.c

@@ -13,23 +13,14 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TDB_HEAP_H_
-#define _TD_TDB_HEAP_H_
+#include "tdb_db.h"
 
-#include "tdbDef.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
+int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) {
   // TODO
-} TDB_HEAP;
-
-TDB_PUBLIC int tdbInitHeapDB(TDB *dbp);
-
-#ifdef __cplusplus
+  return 0;
 }
-#endif
 
-#endif /*_TD_TDB_HEAP_H_*/
\ No newline at end of file
+int tdbClose(TDB *dbp, uint32_t flags) {
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdb_mpool.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdb_mpool.h"
+
+static int         tdbGnrtFileID(const char *fname, uint8_t *fileid);
+static void        tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
+static void        tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
+static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid);
+static int         tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p);
+static int         tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p);
+static void        tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp);
+
+int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) {
+  TDB_MPOOL *mp = NULL;
+  size_t     tsize;
+  pg_t *     pagep;
+
+  // check parameters
+  if (!TDB_IS_PGSIZE_VLD(pgsize)) {
+    tdbError("invalid page size");
+    return -1;
+  }
+
+  // allocate handle
+  mp = (TDB_MPOOL *)calloc(1, sizeof(*mp));
+  if (mp == NULL) {
+    tdbError("failed to malloc memory pool handle");
+    goto _err;
+  }
+
+  // initialize the handle
+  mp->cachesize = cachesize;
+  mp->pgsize = pgsize;
+  mp->npages = cachesize / pgsize;
+  mp->clockHand = 0;
+
+  TD_DLIST_INIT(&mp->freeList);
+
+  mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t));
+  if (mp->pages == NULL) {
+    tdbError("failed to malloc memory pool pages");
+    goto _err;
+  }
+
+  for (frame_id_t i = 0; i < mp->npages; i++) {
+    mp->pages[i].p = malloc(pgsize);
+    if (mp->pages[i].p == NULL) {
+      goto _err;
+    }
+
+    taosInitRWLatch(&mp->pages[i].rwLatch);
+    mp->pages[i].frameid = i;
+    mp->pages[i].pgid = TDB_IVLD_PGID;
+
+    // add new page to the free list
+    TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free);
+  }
+
+#define PGTAB_FACTOR 1.0
+  mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR;
+  mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t));
+  if (mp->pgtab.hashtab == NULL) {
+    tdbError("failed to malloc memory pool hash table");
+    goto _err;
+  }
+
+  // return
+  *mpp = mp;
+  return 0;
+
+_err:
+  tdbMPoolClose(mp);
+  *mpp = NULL;
+  return -1;
+}
+
+int tdbMPoolClose(TDB_MPOOL *mp) {
+  if (mp) {
+    tfree(mp->pgtab.hashtab);
+    if (mp->pages) {
+      for (int i = 0; i < mp->npages; i++) {
+        tfree(mp->pages[i].p);
+      }
+
+      free(mp->pages);
+    }
+
+    free(mp);
+  }
+  return 0;
+}
+
+int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) {
+  TDB_MPFILE *mpf;
+
+  if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) {
+    return -1;
+  }
+
+  mpf->fd = -1;
+
+  if ((mpf->fname = strdup(fname)) == NULL) {
+    goto _err;
+  }
+
+  if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) {
+    goto _err;
+  }
+
+  if (tdbGnrtFileID(fname, mpf->fileid) < 0) {
+    goto _err;
+  }
+
+  // Register current MPF to MP
+  tdbMPoolRegFile(mp, mpf);
+
+  *mpfp = mpf;
+  return 0;
+
+_err:
+  tdbMPoolFileClose(mpf);
+  *mpfp = NULL;
+  return -1;
+}
+
+int tdbMPoolFileClose(TDB_MPFILE *mpf) {
+  if (mpf) {
+    if (mpf->fd > 0) {
+      close(mpf->fd);
+    }
+    tfree(mpf->fname);
+    free(mpf);
+  }
+  return 0;
+}
+
+#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \
+  ({                                                  \
+    uint64_t *tmp = (uint64_t *)fileid;               \
+    (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \
+  })
+
+int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) {
+  // TODO
+  return 0;
+}
+
+int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) {
+  // TODO
+  return 0;
+}
+
+int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
+  pg_t *     pagep;
+  TDB_MPOOL *mp;
+  pg_list_t *pglist;
+
+  mp = mpf->mp;
+
+  // check if the page already in pool
+  pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket);
+  pagep = TD_DLIST_HEAD(pglist);
+  while (pagep) {
+    if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) {
+      break;
+    }
+
+    pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash);
+  }
+
+  if (pagep) {
+    // page is found
+    // todo: pin the page and return
+    *(void **)addr = pagep->p;
+    return 0;
+  }
+
+  // page not found
+  pagep = TD_DLIST_HEAD(&mp->freeList);
+  if (pagep) {
+    // has free page
+    TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free);
+  } else {
+    // no free page available
+    tdbMPoolClockEvictPage(mp, &pagep);
+    if (pagep) {
+      if (pagep->dirty) {
+        // TODO: Handle dirty page eviction
+      }
+    }
+  }
+
+  if (pagep == NULL) {
+    // no available container page
+    return -1;
+  }
+
+  // load page from the disk if a container page is available
+  // TODO: load the page from the disk
+  if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) {
+    return -1;
+  }
+
+  memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN);
+  pagep->pgid.pgno = pgno;
+  pagep->dirty = 0;
+  pagep->pinRef = 1;
+
+  // add current page to page table
+  TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash);
+
+  return 0;
+}
+
+int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
+  // TODO
+  return 0;
+}
+
+static int tdbGnrtFileID(const char *fname, uint8_t *fileid) {
+  struct stat statbuf;
+
+  if (stat(fname, &statbuf) < 0) {
+    return -1;
+  }
+
+  memset(fileid, 0, TDB_FILE_ID_LEN);
+
+  ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino;
+  ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev;
+  ((uint64_t *)fileid)[2] = rand();
+
+  return 0;
+}
+
+#define MPF_GET_BUCKETID(fileid)                   \
+  ({                                               \
+    uint64_t *tmp = (uint64_t *)fileid;            \
+    (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \
+  })
+
+static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
+  mpf_bucket_t *bktp;
+
+  bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid);
+
+  taosWLockLatch(&(bktp->latch));
+
+  TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node);
+
+  taosWUnLockLatch(&(bktp->latch));
+
+  mpf->mp = mp;
+}
+
+static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) {
+  TDB_MPFILE *  mpf = NULL;
+  mpf_bucket_t *bktp;
+
+  bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid);
+
+  taosRLockLatch(&(bktp->latch));
+
+  mpf = TD_DLIST_HEAD(bktp);
+  while (mpf) {
+    if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) {
+      break;
+    }
+
+    mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node);
+  }
+
+  taosRUnLockLatch(&(bktp->latch));
+
+  return mpf;
+}
+
+static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
+  mpf_bucket_t *bktp;
+  TDB_MPFILE *  tmpf;
+
+  if (mpf->mp == NULL) return;
+
+  ASSERT(mpf->mp == mp);
+
+  bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid);
+
+  taosWLockLatch(&(bktp->latch));
+
+  tmpf = TD_DLIST_HEAD(bktp);
+
+  while (tmpf) {
+    if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) {
+      TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node);
+      break;
+    }
+
+    tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node);
+  }
+
+  taosWUnLockLatch(&(bktp->latch));
+
+  ASSERT(tmpf == mpf);
+}
+
+static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) {
+  pgsize_t   pgsize;
+  TDB_MPOOL *mp;
+  off_t      offset;
+  size_t     rsize;
+
+  mp = mpf->mp;
+  pgsize = mp->pgsize;
+  offset = pgno * pgsize;
+
+  // TODO: use loop to read all data
+  rsize = pread(mpf->fd, p, pgsize, offset);
+  // TODO: error handle
+
+  return 0;
+}
+
+static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) {
+  pgsize_t   pgsize;
+  TDB_MPOOL *mp;
+  off_t      offset;
+
+  mp = mpf->mp;
+  pgsize = mp->pgsize;
+  offset = pgno * pgsize;
+
+  lseek(mpf->fd, offset, SEEK_SET);
+  // TODO: handle error
+
+  write(mpf->fd, p, pgsize);
+  // TODO: handle error
+
+  return 0;
+}
+
+static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) {
+  pg_t *     pagep;
+  frame_id_t och;
+
+  *pagepp = NULL;
+  och = mp->clockHand;
+
+  do {
+    pagep = mp->pages + mp->clockHand;
+    mp->clockHand = (mp->clockHand + 1) % mp->npages;
+
+    if (pagep->pinRef == 0) {
+      if (pagep->rbit == 1) {
+        pagep->rbit = 0;
+      } else {
+        break;
+      }
+    }
+
+    if (mp->clockHand == och) {
+      return;
+    }
+  } while (1);
+
+  *pagepp = pagep;
+}
\ No newline at end of file

source/libs/tdb/src/dmgr/tdbDiskMgr.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "tdbDiskMgr.h"
-
-struct STkvDiskMgr {
-  char *   fname;
-  uint16_t pgsize;
-  FileFd   fd;
-  pgid_t   npgid;
-};
-
-#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE))
-
-int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) {
-  STkvDiskMgr *pDiskMgr;
-
-  pDiskMgr = malloc(sizeof(*pDiskMgr));
-  if (pDiskMgr == NULL) {
-    return -1;
-  }
-
-  pDiskMgr->fname = strdup(fname);
-  if (pDiskMgr->fname == NULL) {
-    free(pDiskMgr);
-    return -1;
-  }
-  pDiskMgr->pgsize = pgsize;
-  pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755);
-  if (pDiskMgr->fd < 0) {
-    free(pDiskMgr->fname);
-    free(pDiskMgr);
-    return -1;
-  }
-
-  *ppDiskMgr = pDiskMgr;
-
-  return 0;
-}
-
-int tdmClose(STkvDiskMgr *pDiskMgr) {
-  close(pDiskMgr->fd);
-  free(pDiskMgr->fname);
-  free(pDiskMgr);
-  return 0;
-}
-
-int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) {
-  taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET);
-  taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize);
-  return 0;
-}
-
-int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) {
-  taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET);
-  taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize);
-  return 0;
-}
-
-int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); }
-
-int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; }
\ No newline at end of file

source/libs/tdb/src/inc/tdbBtree.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TDB_BTREE_H_
-#define _TD_TDB_BTREE_H_
-
-#include "tdbDef.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
-  pgid_t root;  // root page number
-} TDB_BTREE;
-
-TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TDB_BTREE_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbBufPool.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TDB_BUF_POOL_H_
-#define _TD_TDB_BUF_POOL_H_
-
-#include "tdbPage.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct STdbBufPool STdbBufPool;
-
-int       tbpOpen(STdbBufPool **ppTkvBufPool);
-int       tbpClose(STdbBufPool *pTkvBufPool);
-STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool);
-int       tbpDelPage(STdbBufPool *pTkvBufPool);
-STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid);
-int       tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid);
-void      tbpFlushPages(STdbBufPool *pTkvBufPool);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TDB_BUF_POOL_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbDiskMgr.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TDISK_MGR_H_
-#define _TD_TDISK_MGR_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "os.h"
-
-#include "tdbDef.h"
-
-typedef struct STkvDiskMgr STkvDiskMgr;
-
-int    tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize);
-int    tdmClose(STkvDiskMgr *pDiskMgr);
-int    tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData);
-int    tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData);
-int    tdmFlush(STkvDiskMgr *pDiskMgr);
-pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TDISK_MGR_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbHash.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TDB_HASH_H_
-#define _TD_TDB_HASH_H_
-
-#include "tdbDef.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
-  // TODO
-} TDB_HASH;
-
-TDB_PUBLIC int tdbInitHashDB(TDB *dbp);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TDB_HASH_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbPage.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TKV_PAGE_H_
-#define _TD_TKV_PAGE_H_
-
-#include "os.h"
-#include "tdbDef.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
-  pgid_t  pgid;
-  int32_t pinCount;
-  bool    idDirty;
-  char*   pData;
-} STdbPage;
-
-typedef struct {
-  uint16_t dbver;
-  uint16_t pgsize;
-  uint32_t cksm;
-} STdbPgHdr;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TKV_PAGE_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbDB.h → source/libs/tdb/src/inc/tdb_db.h

@@ -16,38 +16,28 @@
 #ifndef _TD_TDB_DB_H_
 #define _TD_TDB_DB_H_
 
-#include "tdb.h"
-#include "tdbBtree.h"
-#include "tdbHash.h"
-#include "tdbHeap.h"
+#include "tdb_mpool.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct {
-  // TODO
-} TDB_MPOOL;
-
-typedef struct {
-  int fd;
-} TDB_FH;
+typedef struct TDB TDB;
 
 struct TDB {
-  pgsize_t pageSize;
-  tdb_db_t type;
-  char *   fname;
-  char *   dbname;
-  union {
-    TDB_BTREE *btree;
-    TDB_HASH * hash;
-    TDB_HEAP * heap;
-  } dbam;  // db access method
-
-  TDB_FH *   fhp;  // The backup file handle
-  TDB_MPOOL *mph;  // The memory pool handle
+  char *      fname;
+  char *      dbname;
+  TDB_MPFILE *mpf;
+  // union {
+  //   TDB_BTREE *btree;
+  //   TDB_HASH * hash;
+  //   TDB_HEAP * heap;
+  // } dbam;  // db access method
 };
 
+int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags);
+int tdbClose(TDB *dbp, uint32_t flags);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/tdb/src/inc/tdbDef.h → source/libs/tdb/src/inc/tdb_inc.h

@@ -13,18 +13,30 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TDB_DEF_H_
-#define _TD_TDB_DEF_H_
+#ifndef _TD_TDB_INC_H_
+#define _TD_TDB_INC_H_
 
 #include "os.h"
+#include "tlist.h"
+#include "tlockfree.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+// pgno_t
+typedef int32_t pgno_t;
+#define TDB_IVLD_PGNO ((pgno_t)-1)
+
+// fileid
+#define TDB_FILE_ID_LEN 24
+
 // pgid_t
-typedef int32_t pgid_t;
-#define TDB_IVLD_PGID ((pgid_t)-1)
+typedef struct {
+  uint8_t fileid[TDB_FILE_ID_LEN];
+  pgno_t  pgno;
+} pgid_t;
+#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO};
 
 // framd_id_t
 typedef int32_t frame_id_t;
@@ -34,10 +46,13 @@ typedef int32_t pgsize_t;
 #define TDB_MIN_PGSIZE 512
 #define TDB_MAX_PGSIZE 16384
 #define TDB_DEFAULT_PGSIZE 4096
-#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE))
+#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
+
+// tdb_log
+#define tdbError(var)
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TDB_DEF_H_*/
\ No newline at end of file
+#endif /*_TD_TDB_INC_H_*/

source/libs/tdb/src/inc/tdb_mpool.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TDB_MPOOL_H_
+#define _TD_TDB_MPOOL_H_
+
+#include "tdb_inc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Exposed handle
+typedef struct TDB_MPOOL  TDB_MPOOL;
+typedef struct TDB_MPFILE TDB_MPFILE;
+
+typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t;
+typedef struct pg_t {
+  SRWLatch             rwLatch;
+  frame_id_t           frameid;
+  pgid_t               pgid;
+  uint8_t              dirty;
+  uint8_t              rbit;
+  int32_t              pinRef;
+  pg_free_dlist_node_t free;
+  pg_hash_dlist_node_t hash;
+  void *               p;
+} pg_t;
+
+typedef TD_DLIST(pg_t) pg_list_t;
+typedef struct {
+  SRWLatch latch;
+  TD_DLIST(TDB_MPFILE);
+} mpf_bucket_t;
+struct TDB_MPOOL {
+  int64_t    cachesize;
+  pgsize_t   pgsize;
+  int32_t    npages;
+  pg_t *     pages;
+  pg_list_t  freeList;
+  frame_id_t clockHand;
+  struct {
+    int32_t    nbucket;
+    pg_list_t *hashtab;
+  } pgtab;  // page table, hash<pgid_t, pg_t>
+  struct {
+#define MPF_HASH_BUCKETS 16
+    mpf_bucket_t buckets[MPF_HASH_BUCKETS];
+  } mpfht;  // MPF hash table. MPFs using this MP will be put in this hash table
+};
+
+#define MP_PAGE_AT(mp, idx) (mp)->pages[idx]
+
+typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t;
+struct TDB_MPFILE {
+  char *              fname;                    // file name
+  int                 fd;                       // fd
+  uint8_t             fileid[TDB_FILE_ID_LEN];  // file ID
+  TDB_MPOOL *         mp;                       // underlying memory pool
+  td_mpf_dlist_node_t node;
+};
+
+/*=================================================== Exposed apis ==================================================*/
+// TDB_MPOOL
+int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize);
+int tdbMPoolClose(TDB_MPOOL *mp);
+int tdbMPoolSync(TDB_MPOOL *mp);
+
+// TDB_MPFILE
+int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp);
+int tdbMPoolFileClose(TDB_MPFILE *mpf);
+int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr);
+int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr);
+int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
+int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
+int tdbMPoolFileSync(TDB_MPFILE *mpf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TDB_MPOOL_H_*/
\ No newline at end of file

source/libs/tdb/src/mpool/tdbBufPool.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "thash.h"
-#include "tlist.h"
-
-#include "tdbBufPool.h"
-#include "tdbDiskMgr.h"
-#include "tdbPage.h"
-
-struct SFrameIdWrapper {
-  TD_SLIST_NODE(SFrameIdWrapper);
-  frame_id_t id;
-};
-
-struct STdbBufPool {
-  STdbPage*    pages;
-  STkvDiskMgr* pDiskMgr;
-  SHashObj*    pgTb;  // page_id_t --> frame_id_t
-  TD_SLIST(SFrameIdWrapper) freeList;
-  pthread_mutex_t mutex;
-};
-
-typedef struct STkvLRUReplacer {
-} STkvLRUReplacer;
-
-typedef struct STkvLFUReplacer {
-} STkvLFUReplacer;
-
-typedef struct STkvCLKReplacer {
-} STkvCLKReplacer;
-
-typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t;
-
-typedef struct STkvReplacer {
-  tkv_replacer_t type;
-  union {
-    STkvLRUReplacer lruRep;
-    STkvLFUReplacer lfuRep;
-    STkvCLKReplacer clkRep;
-  };
-} STkvReplacer;
\ No newline at end of file

source/libs/tdb/test/CMakeLists.txt

+# tdbMPoolTest
+add_executable(tdbMPoolTest "tdbMPoolTest.cpp")
+target_link_libraries(tdbMPoolTest tdb gtest gtest_main)
+
 # tdbTest
 add_executable(tdbTest "tdbTest.cpp")
 target_link_libraries(tdbTest tdb gtest gtest_main)
\ No newline at end of file

source/libs/tdb/test/tdbMPoolTest.cpp

+#include "gtest/gtest.h"
+
+#include <iostream>
+
+#include "tdb_mpool.h"
+
+TEST(tdb_mpool_test, test1) {
+  TDB_MPOOL * mp;
+  TDB_MPFILE *mpf;
+  pgno_t      pgno;
+  void *      pgdata;
+
+  // open mp
+  tdbMPoolOpen(&mp, 16384, 4096);
+
+  // open mpf
+  tdbMPoolFileOpen(&mpf, "test.db", mp);
+
+#define TEST1_TOTAL_PAGES 100
+  for (int i = 0; i < TEST1_TOTAL_PAGES; i++) {
+    tdbMPoolFileNewPage(mpf, &pgno, pgdata);
+
+    *(pgno_t *)pgdata = i;
+  }
+
+  // close mpf
+  tdbMPoolFileClose(mpf);
+
+  // close mp
+  tdbMPoolClose(mp);
+}

source/libs/tdb/test/tdbTest.cpp

@@ -3,8 +3,8 @@
 #include "tdb.h"
 
 TEST(tdb_api_test, tdb_create_open_close_db_test) {
-  int  ret;
-  TDB *dbp;
+  // int  ret;
+  // TDB *dbp;
 
   // tdbCreateDB(&dbp, TDB_BTREE_T);