Merge pull request #15002 from taosdata/fix/tdb-multidb

fix/tdb: first round multi-db implementation

source/libs/tdb/src/db/tdbBtree.c

@@ -67,7 +67,8 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
 static int tdbBtcMoveDownward(SBTC *pBtc);
 static int tdbBtcMoveUpward(SBTC *pBtc);
 
-int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SBTree **ppBt) {
+int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr,
+                 SBTree **ppBt) {
   SBTree *pBt;
   int     ret;
 
@@ -99,13 +100,54 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SB
   // pBt->minLeaf
   pBt->minLeaf = pBt->minLocal;
 
+  // if pgno == 0 fetch new btree root leaf page
+  if (pgno == 0) {
+    // fetch page & insert into main db
+    // allocate a new child page
+    SPage *pPage;
+    TXN    txn;
+    tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
+
+    pPager->inTran = 1;
+
+    SBtreeInitPageArg zArg;
+    zArg.flags = 0x1 | 0x2;  // root leaf node;
+    zArg.pBt = pBt;
+    ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn);
+    if (ret < 0) {
+      return -1;
+    }
+
+    // TODO: Need to zero the page
+
+    ret = tdbPagerWrite(pPager, pPage);
+    if (ret < 0) {
+      return -1;
+    }
+
+    if (strcmp(TDB_MAINDB_NAME, tbname)) {
+      ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pgno, sizeof(SPgno), &txn);
+      if (ret < 0) {
+        return -1;
+      }
+    }
+
+    // tdbUnrefPage(pPage);
+    tdbPCacheRelease(pPager->pCache, pPage, &txn);
+    tdbCommit(pPager->pEnv, &txn);
+    tdbTxnClose(&txn);
+  }
+
+  ASSERT(pgno != 0);
+  pBt->root = pgno;
+  /*
   // TODO: pBt->root
   ret = tdbBtreeOpenImpl(pBt);
   if (ret < 0) {
     tdbOsFree(pBt);
     return -1;
   }
-
+  */
   *ppBt = pBt;
   return 0;
 }
@@ -338,7 +380,6 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
 
   ASSERT(pgno != 0);
   pBt->root = pgno;
-
   return 0;
 }
 

source/libs/tdb/src/db/tdbDb.c

@@ -64,6 +64,14 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb) {
 
   mkdir(dbname, 0755);
 
+#ifdef USE_MAINDB
+  // open main db
+  ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SPgno), NULL, pDb, &pDb->pMainDb);
+  if (ret < 0) {
+    return -1;
+  }
+#endif
+
   *ppDb = pDb;
   return 0;
 }
@@ -72,6 +80,10 @@ int tdbClose(TDB *pDb) {
   SPager *pPager;
 
   if (pDb) {
+#ifdef USE_MAINDB
+    if (pDb->pMainDb) tdbTbClose(pDb->pMainDb);
+#endif
+
     for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) {
       pDb->pgrList = pPager->pNext;
       tdbPagerClose(pPager);
@@ -179,4 +191,4 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) {
   if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) {
     // TODO
   }
-}
\ No newline at end of file
+}

source/libs/tdb/src/db/tdbPager.c

@@ -174,6 +174,13 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
   for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage);
        ppPage = &((*ppPage)->pDirtyNext)) {
   }
+
+  if (*ppPage && TDB_PAGE_PGNO(*ppPage) == TDB_PAGE_PGNO(pPage)) {
+    tdbUnrefPage(pPage);
+
+    return 0;
+  }
+
   ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage));
   pPage->pDirtyNext = *ppPage;
   *ppPage = pPage;
@@ -467,7 +474,7 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
   }
 
   TXN txn;
-  tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
+  tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
   SBtreeInitPageArg iArg;
   iArg.pBt = pBt;
   iArg.flags = 0;
@@ -487,6 +494,7 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
       return -1;
     }
 
+    /*
     ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &iArg, &txn);
     if (ret < 0) {
       return -1;
@@ -499,6 +507,18 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
     }
 
     tdbPCacheRelease(pPager->pCache, pPage, &txn);
+    */
+    i64 offset = pPager->pageSize * (pgno - 1);
+    if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
+      ASSERT(0);
+      return -1;
+    }
+
+    ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
   }
 
   tdbOsFSync(pPager->fd);

source/libs/tdb/src/db/tdbTable.c

@@ -16,7 +16,7 @@
 #include "tdbInt.h"
 
 struct STTB {
-  TDB *   pEnv;
+  TDB    *pEnv;
   SBTree *pBt;
 };
 
@@ -25,12 +25,16 @@ struct STBC {
 };
 
 int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) {
-  TTB *   pTb;
+  TTB    *pTb;
   SPager *pPager;
   int     ret;
   char    fFullName[TDB_FILENAME_LEN];
-  SPage * pPage;
+  SPage  *pPage;
   SPgno   pgno;
+  void   *pKey = NULL;
+  int     nKey = 0;
+  void   *pData = NULL;
+  int     nData = 0;
 
   *ppTb = NULL;
 
@@ -42,6 +46,48 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
   // pTb->pEnv
   pTb->pEnv = pEnv;
 
+#ifdef USE_MAINDB
+  snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, TDB_MAINDB_NAME);
+
+  if (strcmp(TDB_MAINDB_NAME, tbname)) {
+    pPager = tdbEnvGetPager(pEnv, fFullName);
+    if (!pPager) {
+      return -1;
+    }
+
+    ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData);
+    if (ret < 0) {
+      // new pgno & insert into main db
+      pgno = 0;
+    } else {
+      pgno = *(SPgno *)pData;
+
+      tdbFree(pKey);
+      tdbFree(pData);
+    }
+
+  } else {
+    pPager = tdbEnvGetPager(pEnv, fFullName);
+    if (pPager == NULL) {
+      ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
+      if (ret < 0) {
+        return -1;
+      }
+
+      tdbEnvAddPager(pEnv, pPager);
+
+      pPager->pEnv = pEnv;
+    }
+
+    if (pPager->dbOrigSize > 0) {
+      pgno = 1;
+    } else {
+      pgno = 0;
+    }
+  }
+
+#else
+
   pPager = tdbEnvGetPager(pEnv, tbname);
   if (pPager == NULL) {
     snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname);
@@ -53,10 +99,12 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
     tdbEnvAddPager(pEnv, pPager);
   }
 
+#endif
+
   ASSERT(pPager != NULL);
 
   // pTb->pBt
-  ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pTb->pBt));
+  ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt));
   if (ret < 0) {
     return -1;
   }

source/libs/tdb/src/inc/tdbInt.h

@@ -150,7 +150,8 @@ struct SBTC {
 };
 
 // SBTree
-int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt);
+int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr,
+                 SBTree **ppBt);
 int tdbBtreeClose(SBTree *pBt);
 int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
 int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn);
@@ -356,6 +357,12 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
   return pCell;
 }
 
+#define USE_MAINDB
+
+#ifdef USE_MAINDB
+#define TDB_MAINDB_NAME "main.tdb"
+#endif
+
 struct STDB {
   char    *dbName;
   char    *jnName;
@@ -365,6 +372,9 @@ struct STDB {
   int      nPager;
   int      nPgrHash;
   SPager **pgrHash;
+#ifdef USE_MAINDB
+  TTB *pMainDb;
+#endif
 };
 
 struct SPager {
@@ -381,6 +391,9 @@ struct SPager {
   u8       inTran;
   SPager  *pNext;      // used by TDB
   SPager  *pHashNext;  // used by TDB
+#ifdef USE_MAINDB
+  TDB *pEnv;
+#endif
 };
 
 #ifdef __cplusplus